Skip to content

Commit

Permalink
Improving duplicates
Browse files Browse the repository at this point in the history
  • Loading branch information
@rigon
rigon committed Oct 10, 2023
1 parent d13d6e8 commit ea1830c
Show file tree
Hide file tree
Showing 10 changed files with 164 additions and 73 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,10 @@ Main features:
- [X] Delete photos
- [X] Save favorites
- [X] Easy selection
- [X] Search for duplicates
- [ ] Authentication
- [ ] Photos timeline with virtual scroll
- [ ] View all places from photos in a map
- [ ] Search for duplicates
- [ ] Tool for renaming files
- [ ] Image resizing according with screen

Expand Down
58 changes: 44 additions & 14 deletions server/album.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,16 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"log"
"path/filepath"
"sort"
"strings"
"time"

"github.com/hlubek/readercomp"
"github.com/timshannon/bolthold"
)

type Album struct {
Expand Down Expand Up @@ -202,46 +205,73 @@ type Duplicate struct {
Found []DuplicateFound `json:"found"`
}

func (album *Album) Duplicates(collection *Collection) ([]Duplicate, error) {
func (album *Album) Duplicates(collection *Collection) (map[string]interface{}, error) {
var dups []Duplicate

err := collection.cache.store.ForEach(nil, func(dbPhoto *Photo) error {
for _, photo := range album.photosMap {
for _, photo := range album.photosMap {
total := time.Now()

var size int64 = 0
for _, file := range photo.Files {
size += file.Size
}

var dbPhotos []*Photo
start := time.Now()
err := collection.cache.store.Find(&dbPhotos, bolthold.Where("Size").Eq(size).Index("size"))
fmt.Println("SEARCH:", time.Since(start).String())
//fmt.Println(size, dbPhotos)
if err != nil {
continue
}

var dupFiles []DuplicateFound
for _, dbPhoto := range dbPhotos {
// Same photo, skip
if dbPhoto.Collection == photo.Collection && dbPhoto.Album == photo.Album && dbPhoto.Id == photo.Id {
continue
}

found := false
dup := Duplicate{
Photo: photo,
Found: []DuplicateFound{},
}
for _, dbFile := range dbPhoto.Files {
for _, file := range photo.Files {
if dbFile.Size == file.Size {
start := time.Now()
equal, err := readercomp.FilesEqual(file.Path, dbFile.Path)
if err != nil {
continue
}
fmt.Println("COMPARE:", time.Since(start).String())
//fmt.Println("COMPARE:", file.Path, dbFile.Path, equal)
if equal {
found = true
dup.Found = append(dup.Found, DuplicateFound{
dupFiles = append(dupFiles, DuplicateFound{
Collection: dbPhoto.Collection,
Album: dbPhoto.Album,
Photo: dbPhoto.Id,
File: dbFile.Id,
})
found = true
}
}
}
}
if found {
dups = append(dups, dup)
if !found {
log.Println("Missed duplicate [", photo.Collection, photo.Album, photo.Id, "] - [", dbPhoto.Collection, dbPhoto.Album, dbPhoto.Id, "]")
}
}
return nil
})
if len(dupFiles) > 0 {
dups = append(dups, Duplicate{
Photo: photo,
Found: dupFiles,
})
}
fmt.Println("TOTAL:", time.Since(total).String(), photo.Id)
}

return dups, err
return map[string]interface{}{
"total": len(album.photosMap),
"countDup": len(dups),
"countUniq": len(album.photosMap) - len(dups),
"duplicates": dups,
}, nil
}
2 changes: 1 addition & 1 deletion server/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
)

var dbInfo = DbInfo{
Version: 8,
Version: 9,
}

type DbInfo struct {
Expand Down
12 changes: 6 additions & 6 deletions server/file.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@ type File struct {
Id string `json:"id"`
Type string `json:"type"`
MIME string `json:"mime"`
Width int `json:"width"` // Image Width
Height int `json:"height"` // Image Height
Date time.Time `json:"date"` // Image Date taken
Location GPSLocation `json:"-"` // Image location
Orientation Orientation `json:"-"` // Image orientation
Size int64 `json:"-" boltholdIndex:"size"` // Image file size, used to find duplicates
Width int `json:"width"` // Image Width
Height int `json:"height"` // Image Height
Date time.Time `json:"date"` // Image Date taken
Location GPSLocation `json:"-"` // Image location
Orientation Orientation `json:"-"` // Image orientation
Size int64 `json:"-"` // Image file size
}

type FileExtendedInfo struct {
Expand Down
14 changes: 10 additions & 4 deletions server/photo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ type Photo struct {
Location GPSLocation `json:"location" boltholdIndex:"location"`
Files []*File `json:"files"`
HasThumb bool `json:"-" boltholdIndex:"hasthumb"` // Indicates if the thumbnail was generated
Size int64 `json:"-" boltholdIndex:"size"` // Photo total size, used to find duplicates
}

// Add pseudo album to the favorites list
Expand Down Expand Up @@ -140,22 +141,27 @@ func (photo *Photo) GetThumbnail(collection *Collection, album *Album, w io.Writ
func (photo *Photo) FillInfo() error {
var countImages = 0
var countVideos = 0
var size int64 = 0
for _, file := range photo.Files {
// Type
switch file.Type {
case "image":
countImages++
case "video":
countVideos++
}
// Size
size += file.Size
}
photo.Size = size

// Determine photo type
size := len(photo.Files)
if size == 1 {
nfiles := len(photo.Files)
switch {
case nfiles == 1:
file := photo.Files[0]
photo.Type = file.Type
}
if size > 1 {
case nfiles > 1:
if countImages > 0 && countVideos > 0 {
photo.Type = "live"
} else {
Expand Down
14 changes: 8 additions & 6 deletions src/Toolbar.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,16 +142,18 @@ const ToolbarMenu : FC = () => {

return (
<ToolbarProvider>
{inAlbum && <>
<Upload />
<ToolbarItem
{inAlbum && [
(<Upload key="upload"/>),
(<ToolbarItem
key="duplicates"
onClick={() => dialog.duplicates(collection, album)}
icon={<DifferenceIcon />}
title="Duplicates"
tooltip="Find duplicated photos in this album"
aria-label="duplicates" />
<Divider />
</>}
aria-label="duplicates" />),
(<Divider key="div"/>),
]}

<ToolbarItem
onClick={() => dialog.newAlbum()}
icon={<AddAlbumIcon />}
Expand Down
99 changes: 76 additions & 23 deletions src/dialogs/Duplicates.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { SxProps, Theme } from "@mui/material/styles";
import Avatar from '@mui/material/Avatar';
import Box from '@mui/material/Box';
import Button from '@mui/material/Button';
import CloseIcon from '@mui/icons-material/Close';
import DeleteForeverIcon from '@mui/icons-material/DeleteForever';
import Dialog from '@mui/material/Dialog';
import DialogActions from '@mui/material/DialogActions';
Expand All @@ -11,16 +12,19 @@ import DialogContentText from '@mui/material/DialogContentText';
import DialogTitle from '@mui/material/DialogTitle';
import DriveFileMoveIcon from '@mui/icons-material/DriveFileMove';
import Divider from '@mui/material/Divider';
import FavoriteIcon from '@mui/icons-material/Favorite';
import IconButton from '@mui/material/IconButton';
import LinearProgress from '@mui/material/LinearProgress';
import List from '@mui/material/List';
import ListItem from '@mui/material/ListItem';
import ListItemAvatar from '@mui/material/ListItemAvatar';
import ListItemText from '@mui/material/ListItemText';

import { useDialog } from '.';
import { useDuplicatedPhotosQuery } from '../services/api';
import { QuerySaveFavorite, ResponseDuplicates, useDuplicatedPhotosQuery } from '../services/api';
import { SelectionProvider, Selectable, useSelection } from '../Selection';
import { DuplicatedType, PhotoImageType, urls } from '../types';
import { PhotoImageType, urls } from '../types';
import useFavorite from '../favoriteHook';

const selectedStyle: SxProps<Theme> = {
outline: "5px solid dodgerblue",
Expand All @@ -29,8 +33,10 @@ const selectedStyle: SxProps<Theme> = {
// boxSizing: "border-box",
};

type DuplicatedItem = ResponseDuplicates['duplicates'][0];

interface ItemProps {
item: DuplicatedType;
item: DuplicatedItem;
}

const Item: FC<ItemProps> = ({item}) => {
Expand All @@ -41,7 +47,7 @@ const Item: FC<ItemProps> = ({item}) => {
}

return (
<Selectable<DuplicatedType> item={item} onChange={handleSelect}>
<Selectable<DuplicatedItem> item={item} onChange={handleSelect}>
<ListItem alignItems="flex-start" sx={selected ? selectedStyle : {}}>
<ListItemAvatar>
<Avatar alt={item.photo.title} src={urls.thumb(item.photo)} variant='square' />
Expand All @@ -57,12 +63,12 @@ const Item: FC<ItemProps> = ({item}) => {


interface ListItemsProps {
items: DuplicatedType[];
fn: (cb: () => DuplicatedType[]) => void;
items: DuplicatedItem[];
fn: (cb: () => DuplicatedItem[]) => void;
}

const ListItems: FC<ListItemsProps> = ({items, fn}) => {
const { get } = useSelection<DuplicatedType>();
const { get } = useSelection<DuplicatedItem>();

fn(() => get());

Expand All @@ -81,18 +87,49 @@ interface DialogProps {
onClose: () => void;
}

const defaultData: ResponseDuplicates = {
total: 0,
countDup: 0,
countUniq: 0,
duplicates: []
}

const DuplicatesDialog: FC<DialogProps> = ({open, collection, album, onClose}) => {
const dialog = useDialog();
const { data, isFetching } = useDuplicatedPhotosQuery({ collection, album }, {skip: !open});
const fnRef = useRef<() => DuplicatedType[]>(() => []);
const favorite = useFavorite();
const { data = defaultData, isFetching } = useDuplicatedPhotosQuery({ collection, album }, {skip: !open});
const fnRef = useRef<() => DuplicatedItem[]>(() => []);

const dups = data || [];
const noDups = dups.length === 0;
const noDups = isFetching || data.duplicates.length === 0;

const handleClose = () => {
onClose();
};

const handleFavorite = () => {
const duplicatedData = fnRef.current();
const resultArray: QuerySaveFavorite["saveData"][] = [];

duplicatedData.forEach(data => {
data.found.forEach(item => {
const existingEntry = resultArray.find(entry => entry.collection === item.collection && entry.album === item.album);
if (existingEntry) {
existingEntry.photos.push(item.photo);
} else {
resultArray.push({
collection: item.collection,
album: item.album,
photos: [item.photo]
});
}
});
});

console.log("resultArray", resultArray);

resultArray.forEach(item => favorite.save(item, [], true));
};

const handleMove = () => {
const selection = fnRef.current();
// Create urls for thumbnails
Expand All @@ -109,34 +146,50 @@ const DuplicatesDialog: FC<DialogProps> = ({open, collection, album, onClose}) =
onClose();
};

const handleFn = (cb: () => DuplicatedType[]) => {
const handleFn = (cb: () => DuplicatedItem[]) => {
fnRef.current = cb;
}

return (
<Dialog open={open} onClose={handleClose} maxWidth="sm" fullWidth>
<DialogTitle>Duplicated photos</DialogTitle>
<DialogTitle>
Duplicated photos
<IconButton sx={{ position: 'absolute', right: 8, top: 8 }} onClick={handleClose}>
<CloseIcon />
</IconButton>
</DialogTitle>
<DialogContent>
<DialogContentText>
Select duplicated photos:
</DialogContentText>
{isFetching ? (
// Render progressbar while loading
<Box sx={{ width: '100%' }}>
<LinearProgress />
</Box>
):(
noDups? <>No duplicated photos found in the current album</> : (
<SelectionProvider<DuplicatedType> itemToId={i => i.photo.id}>
<ListItems items={dups} fn={handleFn} />
</SelectionProvider>
)
noDups? <>No duplicated photos found in the current album</> : (<>
<DialogContentText>
The following photos were found in another albums:
</DialogContentText>
<DialogContentText>
<ul>
<li>{data.total} photos in the album</li>
<li>{data.countDup} photos are duplicated in another albums</li>
<li>{data.countUniq} photos are unique in this album</li>
</ul>
</DialogContentText>
<DialogContentText>
Please select which photos you want to bookmark, move or delete:
</DialogContentText>
<SelectionProvider<DuplicatedItem> itemToId={i => `${i.photo.collection}:${i.photo.album}:${i.photo.id}`}>
<ListItems items={data.duplicates} fn={handleFn} />
</SelectionProvider>
</>)
)}
</DialogContent>
<DialogActions>
<Button onClick={handleClose} color='inherit'>Cancel</Button>
<Button onClick={handleMove} color='warning' startIcon={<DriveFileMoveIcon />} variant='contained' disableElevation>Move</Button>
<Button onClick={handleDelete} color='error' startIcon={<DeleteForeverIcon />} variant='contained' disableElevation>Delete</Button>
<Button onClick={handleFavorite} color='primary' startIcon={<FavoriteIcon />} variant='contained' disabled={noDups} disableElevation>Favorite</Button>
<Button onClick={handleMove} color='warning' startIcon={<DriveFileMoveIcon />} variant='contained' disabled={noDups} disableElevation>Move</Button>
<Button onClick={handleDelete} color='error' startIcon={<DeleteForeverIcon />} variant='contained' disabled={noDups} disableElevation>Delete</Button>
</DialogActions>
</Dialog>
);
Expand Down
Loading

0 comments on commit ea1830c

Please sign in to comment.