No better performance for concurring requests

[katerinafrid]

I'm trying to speed up the processing of requests to my pytorch model, but I do not see any improvements compared to the standard sequential processing. Am I doing something wrong?
First, I execute utils.py file, then I start a server by running server.py.

utils.py

class NLUPipeline(transformers.Pipeline):
    def preprocess(self, inputs):
        return self.tokenizer(inputs['premise_list'], inputs['hypothesis_list'], truncation='only_first',
                              max_length=128, padding=True, return_tensors='pt').to(self.device)

    def _sanitize_parameters(self, **kwargs):
        return {}, {}, {}

    def _forward(self, model_inputs):
        model_out = self.model(**model_inputs)
        return model_out.logits

    @classmethod
    def postprocess(cls, model_outputs):
        return model_outputs


class NLURunnable(bentoml.Runnable):
    SUPPORT_NVIDIA_GPU = True
    SUPPORTED_RESOURCES = ('cuda' if torch.cuda.is_available() else 'cpu',)
    SUPPORTS_CPU_MULTI_THREADING = True

    def __init__(self):
        # load the model instance
        self.nlu_model = bentoml.transformers.load_model("nlu_pl:latest",  device=0 if torch.cuda.is_available() else -1)

    @bentoml.Runnable.method(batchable=True, batch_dim=0)
    def predict(self, input_data):
        return self.nlu_model(input_data)





def register_nlu_pipeline():
    TASK_NAME = "zero-shot-classification"
    TASK_DEFINITION = {
        "impl": NLUPipeline,
        "tf": (),
        "pt": (transformers.AutoModelForSequenceClassification,),
        "default": {},
        "type": "text",
    }
    SUPPORTED_TASKS[TASK_NAME] = TASK_DEFINITION

def create_nlu_pipeline(nlu_model_path: str = default_nlu_model_path):
    classifier = transformers.pipeline(
        task="zero-shot-classification",
        model=transformers.AutoModelForSequenceClassification.from_pretrained(
            nlu_model_path
        ),
        tokenizer=transformers.AutoTokenizer.from_pretrained(
            nlu_model_path
        ),
    )
    return classifier

nlu_pl = create_nlu_pipeline('path')
bentoml.transformers.save_model(
       'nlu_pl',
        pipeline=nlu_pl,
        signatures={
            "predict": {
                "batchable": True,
                "batch_dim": 0,
            },
        },
    )

server.py

nlu_model = bentoml.transformers.get("nlu_pl:latest")
nlu_runner = bentoml.Runner(NLURunnable, models=[nlu_model],
                            method_configs={"predict": {"max_batch_size": 16, "max_latency_ms": 600}}
                            )
register_nlu_pipeline()
svc = bentoml.Service("server", runners=[nlu_runner])


class NLURequest(BaseModel):
    premise_list: List[str]
    hypothesis_list: List[str]

@svc.api(input=JSON(pydantic_model=NLURequest), output=JSON())
async def nlu_request(json: NLURequest) -> Dict[str, Any]:
    req_body = {"premise_list": json.premise_list, "hypothesis_list": json.hypothesis_list}
    response = await nlu_runner.predict.async_run(req_body)
    return {"result": response.cpu().numpy()}