Skip to content

Commit

Permalink
Better support for re-running superdeduper (#29)
Browse files Browse the repository at this point in the history
Remove the result column a little earlier in the apply_results. We should fully qualify all column names when joining tables, but this minimal fix works for now.
  • Loading branch information
mbauman authored Apr 19, 2017
1 parent 7c44008 commit 4c4ca5d
Showing 1 changed file with 5 additions and 3 deletions.
8 changes: 5 additions & 3 deletions superdeduper/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,9 @@ def apply_results(con, config):
"FROM {schema}.entity_map "
"RIGHT JOIN {schema}.entries_unique USING(_unique_id)".format(**config))

# Remove the dedupe_id column from entries if it already exists
c.execute("ALTER TABLE {table} DROP COLUMN IF EXISTS dedupe_id".format(**config))

# Merge clusters based upon exact matches of a subset of fields. This can
# be done on the unique table or on the actual entries table, but it's more
# efficient to do it now.
Expand All @@ -432,8 +435,8 @@ def apply_results(con, config):
cols, config['schema'], con)

# Add that integer id back to the unique_entries table
c.execute(
"ALTER TABLE {schema}.entries_unique DROP COLUMN IF EXISTS dedupe_id".format(**config))
c.execute("""ALTER TABLE {schema}.entries_unique
DROP COLUMN IF EXISTS dedupe_id""".format(**config))
c.execute("ALTER TABLE {schema}.entries_unique ADD COLUMN dedupe_id INTEGER".format(**config))
c.execute("UPDATE {schema}.entries_unique u SET dedupe_id = m.canon_id "
"FROM {schema}.map m WHERE u._unique_id = m._unique_id".format(**config))
Expand All @@ -455,7 +458,6 @@ def apply_results(con, config):
cols, config['schema'], con)
con.commit()

c.execute("ALTER TABLE {table} DROP COLUMN IF EXISTS dedupe_id".format(**config))
c.execute("ALTER TABLE {table} ADD COLUMN dedupe_id INTEGER".format(**config))
c.execute("UPDATE {table} u SET dedupe_id = m.dedupe_id "
"FROM {schema}.unique_map m WHERE u.{key} = m.{key}".format(**config))
Expand Down

0 comments on commit 4c4ca5d

Please sign in to comment.