tfc_w2a2 dropping to 9.74% accuracy with runtime weights

[Sekijoju]

Hello,

I am having a problem with using runtime_weights properly. I set the appropriate parameters in the folding config and have a setup for loading the weights, which is similar to the generated driver.
When using the generated runtime weights the accuracy drops to 9.74%.
I flush the old weights and the verification does not complain.

The code for loading is:

def load_runtime_weights(self, flush_accel=True, verify=True):
       """Load any existing runtime-writable weights from the specified dir into the
       appropriate layer of the accelerator. Note that this must be enabled
       during the accelerator build process. The runtime weights directory
       is specified as the class member ``runtime_weight_dir``. Runtime-writable
       weights are provided as one .dat file per layer.

       Parameters
       ----------
       flush_accel: bool
           Run the accelerator with dummy input after weights are written to
           flush any stale weight data in the weight streamer FIFOs.
       verify: bool
           Whether the written weights will be re-read and verified.
       """
       w_filenames = []
       
       if not os.path.isdir(self.runtime_weight_dir):
           print("No weights loaded")
           return
       for (dirpath, dirnames, filenames) in os.walk(self.runtime_weight_dir):
           w_filenames.extend(filenames)
       rt_weight_dict = {}
       for w_filename in w_filenames:
           if w_filename.endswith(".dat"):
               with open(self.runtime_weight_dir + "/" + w_filename, "r") as f:
                   dat = f.read()
           else:
               continue
           layer_w = np.fromiter(
               [int(x, 16) for x in dat.strip().split()], dtype=np.uint32
           )
           layer_ind = int(w_filename.split("_")[1])
           rt_weight_dict[layer_ind] = layer_w
       for layer_ind in rt_weight_dict.keys():
           cand_if_name = "finn_design_wrapper_0/s_axilite_{0}".format(layer_ind)
           if cand_if_name in self.ip_dict.keys():
               layer_mmio = getattr(
                   getattr(self, "finn_design_wrapper_0"),
                   "s_axilite_{0}".format(layer_ind)
               ).mmio
               layer_w = rt_weight_dict[layer_ind]
               layer_mmio.write_mm(0, layer_w.tobytes())
               
               if verify:
                   new_w = np.copy(layer_mmio.array[: layer_w.shape[0]])
                   assert (layer_w == new_w).all()
       if flush_accel:
           # run accelerator to flush any stale weights from weight streamer FIFOs
           self.execute_on_buffers()

This is how I load the accelerator:

runtime_weight_dir = '/home/xilinx/jupyter_notebooks/runtime_weights/'+weight_folder+'/'

_mnist_fc_io_shape_dict = {
    "idt": [DataType.UINT8],
    "odt": [DataType.UINT8],
    "input_dma_name": ["idma0"],
    "output_dma_name": ["odma0"],
    "number_of_external_weights": 0,
    "ishape_normal": [(1, 784)],
    "oshape_normal": [(1, 1)],
    "ishape_folded": [(1, 16, 49)],
    "oshape_folded": [(1, 1, 1)],
    "ishape_packed": [(1, 16, 49)],
    "oshape_packed": [(1, 1, 1)],
    "num_inputs" : 1,
    "num_outputs" : 1,
}

def resolve_target_platform(target_platform):
    if target_platform is None:
        return pynq.Device.active_device.name
    else:
        assert target_platform in [x.name for x in pynq.Device.devices]
        return target_platform

def tfc_w2a2_mnist(target_platform=None):
    target_platform = resolve_target_platform(target_platform)
    driver_mode = "zynq-iodma"
    model_name = "tfc-w2a2"
    
    #filename = find_bitfile(model_name, target_platform)
    return FINNExampleOverlay(filename,
                              driver_mode,
                              _mnist_fc_io_shape_dict,
                              batch_size=1000,
                              runtime_weight_dir = runtime_weight_dir)

accel = tfc_w2a2_mnist()
print("Ready")

And finally this is how I test the network:

test_imgs = testx
test_labels = testy

bsize = 1000
ok = 0
nok = 0
total = test_imgs.shape[0]
n_batches = int(total / bsize)

test_imgs = test_imgs.reshape(n_batches, bsize, -1)
test_labels = test_labels.reshape(n_batches, bsize)

for i in range(n_batches):
        ibuf_normal = test_imgs[i].reshape(accel.ibuf_packed_device[0].shape)
        exp = test_labels[i]
        accel.copy_input_data_to_device(ibuf_normal)
        accel.execute_on_buffers()
        obuf_normal = np.empty_like(accel.obuf_packed_device[0])
        accel.copy_output_data_from_device(obuf_normal)
        ret = np.bincount(obuf_normal.flatten() == exp.flatten())
        nok += ret[0]
        ok += ret[1]
        print("batch %d / %d : total OK %d NOK %d" % (i + 1, n_batches, ok, nok))
        
acc = 100.0 * ok / (total)
print("Final accuracy: {}%".format(acc))   

Does anyone have experience with utilizing runtime weights? I am lost here.

Sincerely

[shakeelakram00]

@Sekijoju I am also currently working on using the runtime weights. Would you mind helping me regarding how actually we can use the runtime weights, in my case currently I don't bother about the accuracy drop, I just want to learn how actually I can setup using runtime weights. The only information I have currently is that we need a .dat file for the layers and we have to enable runtime weights during builds.

I would be very thankful if you could help me that how I can enable the runtime weights, generate .dat file and setup using runtime weights during testing.
And if you found the solution of accuracy drop, it would be more useful.
Thanks.

[Sekijoju]

You can enable runtime weights by setting "runtime_writeable_weights" to 1 for all Thresholding_Batch and MatrixVectorActivation in your folding configuration.
In the folding configuration I had used in my original post, I forgot to set the attribute for the Thresholding_Batch, which resulted in the low accuracy after loading new weights. The .dat files are generated along with the driver. Then, in your jupyter notebook, you need to set the runtime_weight_dir of the FINN overlay to the folder containing the .dat files.

I think currently the only way to generate the .dat files for a new weights with the same architecture is to run FINN again with the new model. Below is a python script to generate the .dat files from a FINN compatible onnx, but I do not guarantee it works with the current version. It requires you to run FINN for the model once, then you can generate .dat files for new weights (e.g. after retraining the model):

from finn.custom_op.registry import getCustomOp
from finn.transformation.streamline import Streamline
from finn.transformation.fpgadataflow import convert_to_hls_layers
from qonnx.core.modelwrapper import ModelWrapper
from qonnx.transformation.general import GiveUniqueNodeNames

base_path = "PATH TO MODEL FOLDER (the one containing build.py)"
im_folder = base_path+"PATH TO INTERMEDIATE MODELS"
model = ModelWrapper(im_folder+"step_make_pynq_driver.onnx")

fcl = model.get_nodes_by_op_type("StreamingDataflowPartition")[1]
partition_path = getCustomOp(fcl).get_nodeattr("model")
print(partition_path)
model = ModelWrapper(base_path+partition_path)

#new model
from qonnx.util.cleanup import cleanup_model
from finn.transformation.qonnx.convert_qonnx_to_finn import ConvertQONNXtoFINN
new_model = ModelWrapper("NEW MODEL.ONNX")
#necessary transformations
from qonnx.transformation.fold_constants import FoldConstants
from qonnx.transformation.infer_shapes import InferShapes
from qonnx.transformation.infer_datatypes import InferDataTypes
from qonnx.transformation.general import (
    ApplyConfig,
    GiveReadableTensorNames,
    GiveUniqueNodeNames,
    RemoveStaticGraphInputs,
    RemoveUnusedTensors,
)

print("Step 1: Cleaning up model")
new_model = cleanup_model(new_model)
print("Step 2: Applying transformations")
new_model = new_model.transform(ConvertQONNXtoFINN())
new_model = new_model.transform(InferShapes())
new_model = new_model.transform(FoldConstants())
new_model = new_model.transform(GiveUniqueNodeNames())
new_model = new_model.transform(GiveReadableTensorNames())
new_model = new_model.transform(InferDataTypes())
new_model = new_model.transform(RemoveStaticGraphInputs())

#streamline model
from qonnx.transformation.bipolar_to_xnor import ConvertBipolarMatMulToXnorPopcount
import finn.transformation.streamline.absorb as absorb
from finn.transformation.streamline.reorder import MakeMaxPoolNHWC
from qonnx.transformation.infer_data_layouts import InferDataLayouts
from qonnx.transformation.lower_convs_to_matmul import LowerConvsToMatMul

print("Step 3: Streamlining")
new_model = new_model.transform(absorb.AbsorbSignBiasIntoMultiThreshold())
new_model = new_model.transform(Streamline())
need_lowering = len(new_model.get_nodes_by_op_type("Conv")) > 0
if need_lowering:
    print("Needs lowering")
    new_model = new_model.transform(LowerConvsToMatMul())
    new_model = new_model.transform(MakeMaxPoolNHWC())
    new_model = new_model.transform(absorb.AbsorbTransposeIntoMultiThreshold())
    new_model = new_model.transform(MakeMaxPoolNHWC())
    new_model = new_model.transform(absorb.AbsorbConsecutiveTransposes())
new_model = new_model.transform(ConvertBipolarMatMulToXnorPopcount())
new_model = new_model.transform(Streamline())
# absorb final add-mul nodes into TopK
new_model = new_model.transform(absorb.AbsorbScalarMulAddIntoTopK())
new_model = new_model.transform(InferDataLayouts())
new_model = new_model.transform(RemoveUnusedTensors())

from finn.transformation.move_reshape import RemoveCNVtoFCFlatten
print("Step 4: Convert to HLS Layers")
mem_mode = "decoupled"
new_model = new_model.transform(convert_to_hls_layers.InferThresholdingLayer())
new_model = new_model.transform(convert_to_hls_layers.InferBinaryMatrixVectorActivation(mem_mode))
new_model = new_model.transform(convert_to_hls_layers.InferQuantizedMatrixVectorActivation(mem_mode))
new_model = new_model.transform(convert_to_hls_layers.InferLabelSelectLayer())
new_model = new_model.transform(convert_to_hls_layers.InferThresholdingLayer())
need_conv = len(new_model.get_nodes_by_op_type("Im2Col")) > 0
if need_conv:
    new_model = new_model.transform(convert_to_hls_layers.InferConvInpGen())
    new_model = new_model.transform(convert_to_hls_layers.InferStreamingMaxPool())
    new_model = new_model.transform(RemoveCNVtoFCFlatten())
# get rid of Tranpose -> Tranpose identity seq
new_model = new_model.transform(absorb.AbsorbConsecutiveTransposes())
new_model = new_model.transform(GiveUniqueNodeNames())
new_model = new_model.transform(InferDataLayouts())

weights_dir = "WEIGHTS OUTPUT DIR"
rt_layer_ind = 0
ws_index = 0
th_index = 0
for node in model.graph.node:
    if node.op_type in ["MatrixVectorActivation", "Thresholding_Batch"]:
        node_inst = getCustomOp(node)
        is_rt_weights = node_inst.get_nodeattr("runtime_writeable_weights")
        if is_rt_weights == 1:
            if node.op_type == "MatrixVectorActivation":
                new_layer = new_model.get_nodes_by_op_type(node.op_type)[ws_index]
            else:
                new_layer = new_model.get_nodes_by_op_type(node.op_type)[th_index]
            fcl_w = new_model.get_initializer(new_layer.input[1])
            w_filename = weights_dir + "/%d_%d_%s.dat" % (
                1,
                rt_layer_ind,
                node.name,
            )
            node_inst.make_weight_file(
                fcl_w, "decoupled_runtime", w_filename
            )
            #this is required to get the correct layer from the new model
            #for the old model you could just loop over all nodes
            if node.op_type == "MatrixVectorActivation":
                ws_index += 1
            else:
                th_index += 1
            #increment global index
            rt_layer_ind += 1
    else:
        continue

I hope this helps

[shakeelakram00]

Hi,
Thanks for the replay, I will try work in this way and let's see how it goes.