WIP: RefdesRememberer

BUG=Issue #4, Issue #19
TEST=git checkout c718fbc~1 examples/servo_micro.py
Refdeses should stay the same

examples/servo_micro.refdes_mapping

@@ -1,70 +1,71 @@
-CN1 CN?h4bbf014
-D1 D?h81a480d
-U3 U?h98f08e4
-C9 C?h1edb8e1
-C10 C?h6f489d2
-C12 C?h0ddc584
-C15 C?he21e030
-U7 U?ha52c1e2
-D2 D?h0540962
-C22 C?hfb0b218
-C19 C?hb3fb5d1
-U6 U?h5670bf1
-C2 C?h4f535fb
-C5 C?hccbe1cd
-C1 C?hc5a9879
-FB1 L?hd4fa3f7
-C4 C?h1d24bb0
-C7 C?h65ea8b6
-C6 C?h252b2f9
-C13 C?hd0f94be
-CN2 CN?h0e88b23
-C8 C?h5701139
-Q4 Q?h0a35173
-R22 R?h483064b
-R23 R?hf86a3f9
-J3 CN?h59a4754
-U9 U?hc3af7d3
-C25 C?h29b6c7a
-R502 R?haf51894
-R501 R?h2e8a059
-C20 C?h8e9cde3
-U14 U?h26c5927
-C33 C?h6021be3
-R48 R?h6021be3
-TP1 TP?h6f2b349
-TP2 TP?h429d8f7
-C29 C?hb8c64f5
-R46 R?hf05a5f6
-U56 U?he76e8f9
-C109 C?h3e11e23
-C108 C?h16ae1cc
-U55 U?h1558101
-C111 C?h870507a
-C110 C?h1b34cc9
-U1 U?ha63c2dc
-C11 C?h7fd822d
-U10 U?h1d0b4c9
-C3 C?h0f31962
-U57 U?hc2875ac
-U59 U?he683b0e
-U43 U?h7d19a4f
-U42 U?h128a14f
-R95 R?hcd81fd2
-R96 R?hcd81fd2
-C107 C?h28cc4a3
-C106 C?h4e20e72
-U23 U?h7d19a4f
-U24 U?h128a14f
-R61 R?hcd81fd2
-R60 R?hcd81fd2
-C120 C?h28cc4a3
-C119 C?h4e20e72
-U5 U?h69f7b57
-C14 C?h0f9a910
-U45 U?h5374ffe
-U4 U?h5b22328
-C76 C?hd2273c5
-C77 C?h415c632
-C91 C?hd2273c5
-C90 C?h415c632
+refdes	_refdes_from_context	variable_name	class	value
+CN1	CN?h4bbf014	usb	<class 'servo_micro.UsbConnector'>	1981568-1
+D1	D?h81a480d	usb_esd	<class 'servo_micro.UsbEsdDiode'>	TPD2E001DRLR
+U3	U?h98f08e4	reg3300	<class 'servo_micro.Regulator'>	MIC5504-3.3YMT
+C9	C?h1edb8e1	self	<class 'pcbdl.small_parts.C'>	2.2uF
+C10	C?h6f489d2		<class 'pcbdl.small_parts.C'>	10uF
+C12	C?h0ddc584		<class 'pcbdl.small_parts.C'>	100nF
+C15	C?he21e030		<class 'pcbdl.small_parts.C'>	1000pF
+U7	U?ha52c1e2	reg1800	<class 'servo_micro.Regulator'>	TLV70018DSER
+D2	D?h0540962	drop_diode	<class 'servo_micro.DoubleDiode'>	240-800MV
+C22	C?hfb0b218		<class 'pcbdl.small_parts.C'>	100nF
+C19	C?hb3fb5d1		<class 'pcbdl.small_parts.C'>	1uF
+U6	U?h5670bf1	ec	<class 'servo_micro.ServoEC'>	STM32F072CBU6TR
+C2	C?h4f535fb		<class 'pcbdl.small_parts.C'>	100nF
+C5	C?hccbe1cd		<class 'pcbdl.small_parts.C'>	100nF
+C1	C?hc5a9879		<class 'pcbdl.small_parts.C'>	4.7uF
+FB1	L?hd4fa3f7		<class 'pcbdl.small_parts.L'>	600@100MHzH
+C4	C?h1d24bb0		<class 'pcbdl.small_parts.C'>	1uF
+C7	C?h65ea8b6		<class 'pcbdl.small_parts.C'>	100pF
+C6	C?h252b2f9		<class 'pcbdl.small_parts.C'>	100nF
+C13	C?hd0f94be		<class 'pcbdl.small_parts.C'>	4.7uF
+CN2	CN?h0e88b23	prog	<class 'servo_micro.ProgrammingConnector'>	FH34SRJ-8S-0.5SH(50)
+C8	C?h5701139		<class 'pcbdl.small_parts.C'>	100nF
+Q4	Q?h0a35173	boot0_q	<class 'pcbdl.small_parts.FET'>	CSD13381F4
+R22	R?h483064b		<class 'pcbdl.small_parts.R'>	51.1kΩ
+R23	R?hf86a3f9		<class 'pcbdl.small_parts.R'>	51.1kΩ
+J3	CN?h59a4754	dut	<class 'servo_micro.ServoConnector'>	AXK850145WG
+U9	U?hc3af7d3	io	<class 'servo_micro.I2cIoExpander'>	TCA6416ARTWR
+C25	C?h29b6c7a		<class 'pcbdl.small_parts.C'>	100nF
+R502	R?haf51894		<class 'pcbdl.small_parts.R'>	4.7kΩ
+R501	R?h2e8a059		<class 'pcbdl.small_parts.R'>	4.7kΩ
+C20	C?h8e9cde3		<class 'pcbdl.small_parts.C'>	100nF
+U14	U?h26c5927	mfg_mode_shifter	<class 'servo_micro.LevelShifter1'>	SN74AVC1T45DRLR
+C33	C?h6021be3		<class 'pcbdl.small_parts.C'>	100nF
+R48	R?h6021be3		<class 'pcbdl.small_parts.R'>	4.7kΩ
+TP1	TP?h6f2b349	self	<class 'pcbdl.small_parts.TP'>	TP
+TP2	TP?h429d8f7		<class 'pcbdl.small_parts.TP'>	TP
+C29	C?hb8c64f5		<class 'pcbdl.small_parts.C'>	100nF
+R46	R?hf05a5f6		<class 'pcbdl.small_parts.R'>	0Ω
+U56	U?he76e8f9	shifter1	<class 'servo_micro.LevelShifter4'>	SN74AVC4T774RSVR
+C109	C?h3e11e23		<class 'pcbdl.small_parts.C'>	100nF
+C108	C?h16ae1cc		<class 'pcbdl.small_parts.C'>	100nF
+U55	U?h1558101	shifter2	<class 'servo_micro.LevelShifter4'>	SN74AVC4T774RSVR
+C111	C?h870507a		<class 'pcbdl.small_parts.C'>	100nF
+C110	C?h1b34cc9		<class 'pcbdl.small_parts.C'>	100nF
+U1	U?ha63c2dc	jtag_mux	<class 'servo_micro.Mux'>	313-00929-00
+C11	C?h7fd822d		<class 'pcbdl.small_parts.C'>	100nF
+U10	U?h1d0b4c9	jtag_output_buffer	<class 'servo_micro.OutputBuffer'>	SN74LVC1G126YZPR
+C3	C?h0f31962		<class 'pcbdl.small_parts.C'>	100nF
+U57	U?hc2875ac	s	<class 'servo_micro.LevelShifter4'>	SN74AVC4T774RSVR
+U59	U?he683b0e	s	<class 'servo_micro.LevelShifter4'>	SN74AVC4T774RSVR
+U43	U?h7d19a4f	power_switch	<class 'servo_micro.PowerSwitch'>	ADP194ACBZ-R7
+U42	U?h128a14f	power_switch	<class 'servo_micro.PowerSwitch'>	ADP194ACBZ-R7
+R95	R?hcd81fd2		<class 'pcbdl.small_parts.R'>	4.7kΩ
+R96	R?hcd81fd2		<class 'pcbdl.small_parts.R'>	4.7kΩ
+C107	C?h28cc4a3		<class 'pcbdl.small_parts.C'>	100nF
+C106	C?h4e20e72		<class 'pcbdl.small_parts.C'>	100nF
+U23	U?h7d19a4f	power_switch	<class 'servo_micro.PowerSwitch'>	ADP194ACBZ-R7
+U24	U?h128a14f	power_switch	<class 'servo_micro.PowerSwitch'>	ADP194ACBZ-R7
+R61	R?hcd81fd2		<class 'pcbdl.small_parts.R'>	4.7kΩ
+R60	R?hcd81fd2		<class 'pcbdl.small_parts.R'>	4.7kΩ
+C120	C?h28cc4a3		<class 'pcbdl.small_parts.C'>	100nF
+C119	C?h4e20e72		<class 'pcbdl.small_parts.C'>	100nF
+U5	U?h69f7b57	spi1_mux	<class 'servo_micro.AnalogSwitch'>	TS3A24159
+C14	C?h0f9a910		<class 'pcbdl.small_parts.C'>	100nF
+U45	U?h5374ffe	s	<class 'servo_micro.LevelShifter2'>	SN74AVC2T245RSWR
+U4	U?h5b22328	s	<class 'servo_micro.LevelShifter2'>	SN74AVC2T245RSWR
+C76	C?hd2273c5		<class 'pcbdl.small_parts.C'>	100nF
+C77	C?h415c632		<class 'pcbdl.small_parts.C'>	100nF
+C91	C?hd2273c5		<class 'pcbdl.small_parts.C'>	100nF
+C90	C?h415c632		<class 'pcbdl.small_parts.C'>	100nF

pcbdl/context.py

@@ -14,11 +14,106 @@
 
 from .base import Net, Part, Plugin
 import collections
+import csv
+
 __all__ = [
     "Context",
     "global_context", "nets",
 ]
 
+class RefdesRememberer:
+    """Remembers refdeses from old runs of the schematic. This tries to guarantee that as the schematic evolves, automatically generated refdefs are deterministic. It does this by storing the refdefs in a file (.refdes_mapping), in part definition order, together with some information about the part (called anchors). Even partial matches of the anchors will be enough to remember the name."""
+    anchor_names = ('_refdes_from_context', 'variable_name', 'class', 'value')
+
+    """[(refdes, {anchor_name: anchor})]"""
+    _mapping = []
+
+    csv.register_dialect("pcbdl", delimiter="\t", lineterminator="\n", strict=True) #TODO: strict=False
+
+    class MatchNotFound(Exception):
+        pass
+
+    def read(self):
+        self._mapping = []
+        try:
+            with open(self.filename, "r") as f:
+                reader = csv.DictReader(f, dialect="pcbdl")
+                for row in reader:
+                    refdes = row.pop("refdes")
+                    self._mapping.append((refdes, row))
+        except FileNotFoundError:
+            pass # We'll start fresh!
+
+    def __init__(self, filename):
+        self.filename = filename
+        self.read()
+
+    def find_match(self, part, score_threshold=0.6):
+        current_anchors = self.get_part_anchors(part)
+        max_score = len(self.anchor_names)
+
+        scored_others = [] # score, (refdes, anchors)
+        for refdes, older_anchors in self._mapping:
+            score = 0
+
+            for anchor_name in current_anchors.keys():
+                if anchor_name not in older_anchors:
+                    continue # change of schema?
+
+                if current_anchors[anchor_name] == older_anchors[anchor_name]:
+                    score += 1
+
+            scored_others.append((score, (refdes, older_anchors)))
+
+        scored_others.sort(key=(lambda other: other[0]), reverse=True)
+        first_match = scored_others[0]
+        score, row = first_match
+        if score < max_score * score_threshold:
+            raise self.MatchNotFound()
+
+        refdes, older_anchors = row
+
+        # some logging if it's inexact
+        if score != max_score:
+            print(f"Inexact match ({score}/{max_score}) for {part}:")
+            for anchor_name in current_anchors.keys():
+                if anchor_name not in older_anchors:
+                    print("[%r] %s not found" % (anchor_name, current_anchors[anchor_name]))
+                    continue # change of schema?
+                if current_anchors[anchor_name] != older_anchors[anchor_name]:
+                    print("[%r] %s!=%s" % (anchor_name, current_anchors[anchor_name], older_anchors[anchor_name]))
+
+        #make sure nobody else matches with this row again, since we already found the instance matching it
+        self._mapping.remove(row)
+
+        return refdes
+
+    def get_part_anchors(self, part):
+        anchors = {}
+        #anchors["nets"] =
+        anchors["_refdes_from_context"] = part._refdes_from_context
+        try:
+            anchors["variable_name"] = part.variable_name
+        except AttributeError:
+            anchors["variable_name"] = ""
+        anchors["class"] = repr(part.__class__)
+        anchors["value"] = part.value
+
+        assert(set(anchors.keys()) == set(self.anchor_names))
+        return anchors
+
+    def overwrite(self, context, read_again=True):
+        with open(self.filename, "w") as f:
+            writer = csv.DictWriter(f, dialect="pcbdl", fieldnames=("refdes",) + self.anchor_names)
+            writer.writeheader()
+            for refdes, part in context.named_parts.items():
+                row = self.get_part_anchors(part)
+                row["refdes"] = refdes
+                writer.writerow(row)
+
+        if read_again:
+            self.read() # TODO: nobody really needs this, it's just for _mapping consistency, but there must be a better way (and more perfomant)
+
 class Context(object):
     def __init__(self, name = ""):
         self.name = name
@@ -67,11 +162,7 @@ def name_part_with_mapping(self, part, mapping):
         mapping.pop(i)
 
     def autoname(self, mapping_file=None):
-        try:
-            with open(mapping_file, "r") as file:
-                mapping = [line.strip().split(" ") for line in file.readlines()]
-        except:
-            mapping = []
+        refdes_rememberer = RefdesRememberer(mapping_file)
 
         self.named_parts = collections.OrderedDict()
         for part in self.parts_list:
@@ -81,17 +172,27 @@ def autoname(self, mapping_file=None):
                 number = original_name[len(prefix):]
 
                 if number.startswith("?"):
-                    self.name_part_with_mapping(part, mapping)
-                    number = part.refdes[len(prefix):]
-                    #print ("Renaming %s -> %s with mapping file" % (original_name, part.refdes))
+                    try:
+                        refdes = refdes_rememberer.find_match(part)
+                    except RefdesRememberer.MatchNotFound:
+                        continue
+                    part.refdes = refdes
+                    number = refdes[len(prefix):]
+                    print("Remembering refdes %s -> %s" % (original_name, part.refdes))
+
+        for part in self.parts_list:
+            original_name = part.refdes
+            prefix = part.REFDES_PREFIX
+            if original_name.startswith(prefix):
+                number = original_name[len(prefix):]
 
                 if number.startswith("?"):
                     while True:
                         part.refdes = "%s%d" % (prefix, self.refdes_counters[prefix])
                         self.refdes_counters[prefix] += 1
                         if part.refdes not in self.named_parts:
                             break
-                    #print ("Renaming %s -> %s" % (original_name, part.refdes))
+                    print("New refdes %s -> %s" % (original_name, part.refdes))
                 else:
                     # Yay, there's a part that's already named
                     # Let's remember the number for it and save it
@@ -101,14 +202,12 @@ def autoname(self, mapping_file=None):
                         pass
                     else:
                         self.refdes_counters[prefix] = number
-                        #print ("Skipping ahead to %s%d+1" % (prefix, self.refdes_counters[prefix]))
+                        #print("Skipping ahead to %s%d+1" % (prefix, self.refdes_counters[prefix]))
                         self.refdes_counters[prefix] += 1
             self.named_parts[part.refdes] = part
 
-        if mapping_file:
-            with open(mapping_file, "w") as file:
-                for final_name, part in self.named_parts.items():
-                    file.write("%s %s\n" % (final_name, part._refdes_from_context))
+        refdes_rememberer.overwrite(self, read_again=False)
+        del refdes_rememberer
 
         for net in self.net_list:
             # Look only for unnamed nets