From bd0806e52033ac34178d284a64adfec3df4920fd Mon Sep 17 00:00:00 2001 From: David <1282675518@qq.com> Date: Mon, 4 Sep 2023 17:02:13 +0800 Subject: [PATCH 01/17] update bert4rec --- recbole/model/sequential_recommender/bert4rec.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/recbole/model/sequential_recommender/bert4rec.py b/recbole/model/sequential_recommender/bert4rec.py index 0a759d6ff..03976314c 100644 --- a/recbole/model/sequential_recommender/bert4rec.py +++ b/recbole/model/sequential_recommender/bert4rec.py @@ -75,6 +75,8 @@ def __init__(self, config, dataset): self.LayerNorm = nn.LayerNorm(self.hidden_size, eps=self.layer_norm_eps) self.dropout = nn.Dropout(self.hidden_dropout_prob) + self.output_gelu = nn.GELU() + self.output_bias = nn.Parameter(torch.zeros(self.n_items)) # we only need compute the loss at the masked position try: @@ -124,7 +126,7 @@ def forward(self, item_seq): trm_output = self.trm_encoder( input_emb, extended_attention_mask, output_all_encoded_layers=True ) - output = trm_output[-1] + output = self.output_gelu(trm_output[-1]) return output # [B L H] def multi_hot_embed(self, masked_index, max_length): @@ -172,8 +174,8 @@ def calculate_loss(self, interaction): if self.loss_type == "BPR": pos_items_emb = self.item_embedding(pos_items) # [B mask_len H] neg_items_emb = self.item_embedding(neg_items) # [B mask_len H] - pos_score = torch.sum(seq_output * pos_items_emb, dim=-1) # [B mask_len] - neg_score = torch.sum(seq_output * neg_items_emb, dim=-1) # [B mask_len] + pos_score = torch.sum(seq_output * pos_items_emb, dim=-1) + self.output_bias[pos_items] # [B mask_len] + neg_score = torch.sum(seq_output * neg_items_emb, dim=-1) + self.output_bias[neg_items] # [B mask_len] targets = (masked_index > 0).float() loss = -torch.sum( torch.log(1e-14 + torch.sigmoid(pos_score - neg_score)) * targets @@ -185,7 +187,7 @@ def calculate_loss(self, interaction): test_item_emb = self.item_embedding.weight[: self.n_items] # [item_num H] logits = torch.matmul( seq_output, test_item_emb.transpose(0, 1) - ) # [B mask_len item_num] + ) + self.output_bias # [B mask_len item_num] targets = (masked_index > 0).float().view(-1) # [B*mask_len] loss = torch.sum( @@ -204,7 +206,7 @@ def predict(self, interaction): seq_output = self.forward(item_seq) seq_output = self.gather_indexes(seq_output, item_seq_len - 1) # [B H] test_item_emb = self.item_embedding(test_item) - scores = torch.mul(seq_output, test_item_emb).sum(dim=1) # [B] + scores = (torch.mul(seq_output, test_item_emb)).sum(dim=1) + self.output_bias[test_item] # [B] return scores def full_sort_predict(self, interaction): @@ -218,5 +220,5 @@ def full_sort_predict(self, interaction): ] # delete masked token scores = torch.matmul( seq_output, test_items_emb.transpose(0, 1) - ) # [B, item_num] + ) + self.output_bias # [B, item_num] return scores From 940f36bf89152fb572d5f884541be384253d8ba2 Mon Sep 17 00:00:00 2001 From: David <1282675518@qq.com> Date: Mon, 4 Sep 2023 17:15:34 +0800 Subject: [PATCH 02/17] update bert4rec --- .../model/sequential_recommender/bert4rec.py | 61 ++++++------------- 1 file changed, 17 insertions(+), 44 deletions(-) diff --git a/recbole/model/sequential_recommender/bert4rec.py b/recbole/model/sequential_recommender/bert4rec.py index 03976314c..b6247ea6c 100644 --- a/recbole/model/sequential_recommender/bert4rec.py +++ b/recbole/model/sequential_recommender/bert4rec.py @@ -26,6 +26,7 @@ class BERT4Rec(SequentialRecommender): + def __init__(self, config, dataset): super(BERT4Rec, self).__init__(config, dataset) @@ -33,9 +34,7 @@ def __init__(self, config, dataset): self.n_layers = config["n_layers"] self.n_heads = config["n_heads"] self.hidden_size = config["hidden_size"] # same as embedding_size - self.inner_size = config[ - "inner_size" - ] # the dimensionality in feed-forward layer + self.inner_size = config["inner_size"] # the dimensionality in feed-forward layer self.hidden_dropout_prob = config["hidden_dropout_prob"] self.attn_dropout_prob = config["attn_dropout_prob"] self.hidden_act = config["hidden_act"] @@ -56,12 +55,8 @@ def __init__(self, config, dataset): self.mask_item_length = int(self.mask_ratio * self.max_seq_length) # define layers and loss - self.item_embedding = nn.Embedding( - self.n_items + 1, self.hidden_size, padding_idx=0 - ) # mask token add 1 - self.position_embedding = nn.Embedding( - self.max_seq_length, self.hidden_size - ) # add mask_token at the last + self.item_embedding = nn.Embedding(self.n_items + 1, self.hidden_size, padding_idx=0) # mask token add 1 + self.position_embedding = nn.Embedding(self.max_seq_length, self.hidden_size) # add mask_token at the last self.trm_encoder = TransformerEncoder( n_layers=self.n_layers, n_heads=self.n_heads, @@ -103,9 +98,7 @@ def reconstruct_test_data(self, item_seq, item_seq_len): """ Add mask token at the last position according to the lengths of item_seq """ - padding = torch.zeros( - item_seq.size(0), dtype=torch.long, device=item_seq.device - ) # [B] + padding = torch.zeros(item_seq.size(0), dtype=torch.long, device=item_seq.device) # [B] item_seq = torch.cat((item_seq, padding.unsqueeze(-1)), dim=-1) # [B max_len+1] for batch_id, last_position in enumerate(item_seq_len): item_seq[batch_id][last_position] = self.mask_token @@ -113,9 +106,7 @@ def reconstruct_test_data(self, item_seq, item_seq_len): return item_seq def forward(self, item_seq): - position_ids = torch.arange( - item_seq.size(1), dtype=torch.long, device=item_seq.device - ) + position_ids = torch.arange(item_seq.size(1), dtype=torch.long, device=item_seq.device) position_ids = position_ids.unsqueeze(0).expand_as(item_seq) position_embedding = self.position_embedding(position_ids) item_emb = self.item_embedding(item_seq) @@ -123,9 +114,7 @@ def forward(self, item_seq): input_emb = self.LayerNorm(input_emb) input_emb = self.dropout(input_emb) extended_attention_mask = self.get_attention_mask(item_seq, bidirectional=True) - trm_output = self.trm_encoder( - input_emb, extended_attention_mask, output_all_encoded_layers=True - ) + trm_output = self.trm_encoder(input_emb, extended_attention_mask, output_all_encoded_layers=True) output = self.output_gelu(trm_output[-1]) return output # [B L H] @@ -147,9 +136,7 @@ def multi_hot_embed(self, masked_index, max_length): multi_hot_embed: [[0 1 0 0 0], [0 0 0 1 0]] """ masked_index = masked_index.view(-1) - multi_hot = torch.zeros( - masked_index.size(0), max_length, device=masked_index.device - ) + multi_hot = torch.zeros(masked_index.size(0), max_length, device=masked_index.device) multi_hot[torch.arange(masked_index.size(0)), masked_index] = 1 return multi_hot @@ -160,13 +147,9 @@ def calculate_loss(self, interaction): masked_index = interaction[self.MASK_INDEX] seq_output = self.forward(masked_item_seq) - pred_index_map = self.multi_hot_embed( - masked_index, masked_item_seq.size(-1) - ) # [B*mask_len max_len] + pred_index_map = self.multi_hot_embed(masked_index, masked_item_seq.size(-1)) # [B*mask_len max_len] # [B mask_len] -> [B mask_len max_len] multi hot - pred_index_map = pred_index_map.view( - masked_index.size(0), masked_index.size(1), -1 - ) # [B mask_len max_len] + pred_index_map = pred_index_map.view(masked_index.size(0), masked_index.size(1), -1) # [B mask_len max_len] # [B mask_len max_len] * [B max_len H] -> [B mask_len H] # only calculate loss for masked position seq_output = torch.bmm(pred_index_map, seq_output) # [B mask_len H] @@ -177,23 +160,17 @@ def calculate_loss(self, interaction): pos_score = torch.sum(seq_output * pos_items_emb, dim=-1) + self.output_bias[pos_items] # [B mask_len] neg_score = torch.sum(seq_output * neg_items_emb, dim=-1) + self.output_bias[neg_items] # [B mask_len] targets = (masked_index > 0).float() - loss = -torch.sum( - torch.log(1e-14 + torch.sigmoid(pos_score - neg_score)) * targets - ) / torch.sum(targets) + loss = -torch.sum(torch.log(1e-14 + torch.sigmoid(pos_score - neg_score)) * targets) / torch.sum(targets) return loss elif self.loss_type == "CE": loss_fct = nn.CrossEntropyLoss(reduction="none") - test_item_emb = self.item_embedding.weight[: self.n_items] # [item_num H] - logits = torch.matmul( - seq_output, test_item_emb.transpose(0, 1) - ) + self.output_bias # [B mask_len item_num] + test_item_emb = self.item_embedding.weight[:self.n_items] # [item_num H] + logits = torch.matmul(seq_output, test_item_emb.transpose(0, 1)) + self.output_bias # [B mask_len item_num] targets = (masked_index > 0).float().view(-1) # [B*mask_len] - loss = torch.sum( - loss_fct(logits.view(-1, test_item_emb.size(0)), pos_items.view(-1)) - * targets - ) / torch.sum(targets) + loss = torch.sum(loss_fct(logits.view(-1, test_item_emb.size(0)), pos_items.view(-1)) * targets + ) / torch.sum(targets) return loss else: raise NotImplementedError("Make sure 'loss_type' in ['BPR', 'CE']!") @@ -215,10 +192,6 @@ def full_sort_predict(self, interaction): item_seq = self.reconstruct_test_data(item_seq, item_seq_len) seq_output = self.forward(item_seq) seq_output = self.gather_indexes(seq_output, item_seq_len - 1) # [B H] - test_items_emb = self.item_embedding.weight[ - : self.n_items - ] # delete masked token - scores = torch.matmul( - seq_output, test_items_emb.transpose(0, 1) - ) + self.output_bias # [B, item_num] + test_items_emb = self.item_embedding.weight[:self.n_items] # delete masked token + scores = torch.matmul(seq_output, test_items_emb.transpose(0, 1)) + self.output_bias # [B, item_num] return scores From f411e6b5e2bcb32bf1f734942597b4438a64e07c Mon Sep 17 00:00:00 2001 From: David <1282675518@qq.com> Date: Mon, 4 Sep 2023 17:24:56 +0800 Subject: [PATCH 03/17] add header --- recbole/model/sequential_recommender/bert4rec.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/recbole/model/sequential_recommender/bert4rec.py b/recbole/model/sequential_recommender/bert4rec.py index b6247ea6c..6423a1b83 100644 --- a/recbole/model/sequential_recommender/bert4rec.py +++ b/recbole/model/sequential_recommender/bert4rec.py @@ -3,6 +3,11 @@ # @Author : Hui Wang # @Email : hui.wang@ruc.edu.cn +# UPDATE +# @Time : 2023/9/4 +# @Author : Enze Liu +# @Email : enzeeliu@foxmail.com + r""" BERT4Rec ################################################ From e240ab5d137b2002f6e63cff7202cb784b11c66c Mon Sep 17 00:00:00 2001 From: David <1282675518@qq.com> Date: Sun, 10 Sep 2023 23:16:06 +0800 Subject: [PATCH 04/17] add ffn --- recbole/model/sequential_recommender/bert4rec.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/recbole/model/sequential_recommender/bert4rec.py b/recbole/model/sequential_recommender/bert4rec.py index 6423a1b83..36f143e16 100644 --- a/recbole/model/sequential_recommender/bert4rec.py +++ b/recbole/model/sequential_recommender/bert4rec.py @@ -75,6 +75,7 @@ def __init__(self, config, dataset): self.LayerNorm = nn.LayerNorm(self.hidden_size, eps=self.layer_norm_eps) self.dropout = nn.Dropout(self.hidden_dropout_prob) + self.output_ffn = nn.Linear(self.hidden_size, self.hidden_size) self.output_gelu = nn.GELU() self.output_bias = nn.Parameter(torch.zeros(self.n_items)) @@ -120,7 +121,8 @@ def forward(self, item_seq): input_emb = self.dropout(input_emb) extended_attention_mask = self.get_attention_mask(item_seq, bidirectional=True) trm_output = self.trm_encoder(input_emb, extended_attention_mask, output_all_encoded_layers=True) - output = self.output_gelu(trm_output[-1]) + ffn_output = self.output_ffn(trm_output[-1]) + output = self.output_gelu(ffn_output) return output # [B L H] def multi_hot_embed(self, masked_index, max_length): From e09f196b88bfe85db6e88f271d4c427d54bc8379 Mon Sep 17 00:00:00 2001 From: BishopLiu Date: Sun, 10 Sep 2023 15:38:19 +0000 Subject: [PATCH 05/17] Format Python code according to PEP8 --- .../model/sequential_recommender/bert4rec.py | 76 ++++++++++++++----- 1 file changed, 56 insertions(+), 20 deletions(-) diff --git a/recbole/model/sequential_recommender/bert4rec.py b/recbole/model/sequential_recommender/bert4rec.py index 36f143e16..7554dd80d 100644 --- a/recbole/model/sequential_recommender/bert4rec.py +++ b/recbole/model/sequential_recommender/bert4rec.py @@ -31,7 +31,6 @@ class BERT4Rec(SequentialRecommender): - def __init__(self, config, dataset): super(BERT4Rec, self).__init__(config, dataset) @@ -39,7 +38,9 @@ def __init__(self, config, dataset): self.n_layers = config["n_layers"] self.n_heads = config["n_heads"] self.hidden_size = config["hidden_size"] # same as embedding_size - self.inner_size = config["inner_size"] # the dimensionality in feed-forward layer + self.inner_size = config[ + "inner_size" + ] # the dimensionality in feed-forward layer self.hidden_dropout_prob = config["hidden_dropout_prob"] self.attn_dropout_prob = config["attn_dropout_prob"] self.hidden_act = config["hidden_act"] @@ -60,8 +61,12 @@ def __init__(self, config, dataset): self.mask_item_length = int(self.mask_ratio * self.max_seq_length) # define layers and loss - self.item_embedding = nn.Embedding(self.n_items + 1, self.hidden_size, padding_idx=0) # mask token add 1 - self.position_embedding = nn.Embedding(self.max_seq_length, self.hidden_size) # add mask_token at the last + self.item_embedding = nn.Embedding( + self.n_items + 1, self.hidden_size, padding_idx=0 + ) # mask token add 1 + self.position_embedding = nn.Embedding( + self.max_seq_length, self.hidden_size + ) # add mask_token at the last self.trm_encoder = TransformerEncoder( n_layers=self.n_layers, n_heads=self.n_heads, @@ -104,7 +109,9 @@ def reconstruct_test_data(self, item_seq, item_seq_len): """ Add mask token at the last position according to the lengths of item_seq """ - padding = torch.zeros(item_seq.size(0), dtype=torch.long, device=item_seq.device) # [B] + padding = torch.zeros( + item_seq.size(0), dtype=torch.long, device=item_seq.device + ) # [B] item_seq = torch.cat((item_seq, padding.unsqueeze(-1)), dim=-1) # [B max_len+1] for batch_id, last_position in enumerate(item_seq_len): item_seq[batch_id][last_position] = self.mask_token @@ -112,7 +119,9 @@ def reconstruct_test_data(self, item_seq, item_seq_len): return item_seq def forward(self, item_seq): - position_ids = torch.arange(item_seq.size(1), dtype=torch.long, device=item_seq.device) + position_ids = torch.arange( + item_seq.size(1), dtype=torch.long, device=item_seq.device + ) position_ids = position_ids.unsqueeze(0).expand_as(item_seq) position_embedding = self.position_embedding(position_ids) item_emb = self.item_embedding(item_seq) @@ -120,7 +129,9 @@ def forward(self, item_seq): input_emb = self.LayerNorm(input_emb) input_emb = self.dropout(input_emb) extended_attention_mask = self.get_attention_mask(item_seq, bidirectional=True) - trm_output = self.trm_encoder(input_emb, extended_attention_mask, output_all_encoded_layers=True) + trm_output = self.trm_encoder( + input_emb, extended_attention_mask, output_all_encoded_layers=True + ) ffn_output = self.output_ffn(trm_output[-1]) output = self.output_gelu(ffn_output) return output # [B L H] @@ -143,7 +154,9 @@ def multi_hot_embed(self, masked_index, max_length): multi_hot_embed: [[0 1 0 0 0], [0 0 0 1 0]] """ masked_index = masked_index.view(-1) - multi_hot = torch.zeros(masked_index.size(0), max_length, device=masked_index.device) + multi_hot = torch.zeros( + masked_index.size(0), max_length, device=masked_index.device + ) multi_hot[torch.arange(masked_index.size(0)), masked_index] = 1 return multi_hot @@ -154,9 +167,13 @@ def calculate_loss(self, interaction): masked_index = interaction[self.MASK_INDEX] seq_output = self.forward(masked_item_seq) - pred_index_map = self.multi_hot_embed(masked_index, masked_item_seq.size(-1)) # [B*mask_len max_len] + pred_index_map = self.multi_hot_embed( + masked_index, masked_item_seq.size(-1) + ) # [B*mask_len max_len] # [B mask_len] -> [B mask_len max_len] multi hot - pred_index_map = pred_index_map.view(masked_index.size(0), masked_index.size(1), -1) # [B mask_len max_len] + pred_index_map = pred_index_map.view( + masked_index.size(0), masked_index.size(1), -1 + ) # [B mask_len max_len] # [B mask_len max_len] * [B max_len H] -> [B mask_len H] # only calculate loss for masked position seq_output = torch.bmm(pred_index_map, seq_output) # [B mask_len H] @@ -164,20 +181,33 @@ def calculate_loss(self, interaction): if self.loss_type == "BPR": pos_items_emb = self.item_embedding(pos_items) # [B mask_len H] neg_items_emb = self.item_embedding(neg_items) # [B mask_len H] - pos_score = torch.sum(seq_output * pos_items_emb, dim=-1) + self.output_bias[pos_items] # [B mask_len] - neg_score = torch.sum(seq_output * neg_items_emb, dim=-1) + self.output_bias[neg_items] # [B mask_len] + pos_score = ( + torch.sum(seq_output * pos_items_emb, dim=-1) + + self.output_bias[pos_items] + ) # [B mask_len] + neg_score = ( + torch.sum(seq_output * neg_items_emb, dim=-1) + + self.output_bias[neg_items] + ) # [B mask_len] targets = (masked_index > 0).float() - loss = -torch.sum(torch.log(1e-14 + torch.sigmoid(pos_score - neg_score)) * targets) / torch.sum(targets) + loss = -torch.sum( + torch.log(1e-14 + torch.sigmoid(pos_score - neg_score)) * targets + ) / torch.sum(targets) return loss elif self.loss_type == "CE": loss_fct = nn.CrossEntropyLoss(reduction="none") - test_item_emb = self.item_embedding.weight[:self.n_items] # [item_num H] - logits = torch.matmul(seq_output, test_item_emb.transpose(0, 1)) + self.output_bias # [B mask_len item_num] + test_item_emb = self.item_embedding.weight[: self.n_items] # [item_num H] + logits = ( + torch.matmul(seq_output, test_item_emb.transpose(0, 1)) + + self.output_bias + ) # [B mask_len item_num] targets = (masked_index > 0).float().view(-1) # [B*mask_len] - loss = torch.sum(loss_fct(logits.view(-1, test_item_emb.size(0)), pos_items.view(-1)) * targets - ) / torch.sum(targets) + loss = torch.sum( + loss_fct(logits.view(-1, test_item_emb.size(0)), pos_items.view(-1)) + * targets + ) / torch.sum(targets) return loss else: raise NotImplementedError("Make sure 'loss_type' in ['BPR', 'CE']!") @@ -190,7 +220,9 @@ def predict(self, interaction): seq_output = self.forward(item_seq) seq_output = self.gather_indexes(seq_output, item_seq_len - 1) # [B H] test_item_emb = self.item_embedding(test_item) - scores = (torch.mul(seq_output, test_item_emb)).sum(dim=1) + self.output_bias[test_item] # [B] + scores = (torch.mul(seq_output, test_item_emb)).sum(dim=1) + self.output_bias[ + test_item + ] # [B] return scores def full_sort_predict(self, interaction): @@ -199,6 +231,10 @@ def full_sort_predict(self, interaction): item_seq = self.reconstruct_test_data(item_seq, item_seq_len) seq_output = self.forward(item_seq) seq_output = self.gather_indexes(seq_output, item_seq_len - 1) # [B H] - test_items_emb = self.item_embedding.weight[:self.n_items] # delete masked token - scores = torch.matmul(seq_output, test_items_emb.transpose(0, 1)) + self.output_bias # [B, item_num] + test_items_emb = self.item_embedding.weight[ + : self.n_items + ] # delete masked token + scores = ( + torch.matmul(seq_output, test_items_emb.transpose(0, 1)) + self.output_bias + ) # [B, item_num] return scores From 165b72c31eb3ed1c7ba1736e29a8e908f89458ca Mon Sep 17 00:00:00 2001 From: David <1282675518@qq.com> Date: Wed, 13 Sep 2023 11:09:39 +0800 Subject: [PATCH 06/17] add output layernorm --- recbole/model/sequential_recommender/bert4rec.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recbole/model/sequential_recommender/bert4rec.py b/recbole/model/sequential_recommender/bert4rec.py index 7554dd80d..c936f67b6 100644 --- a/recbole/model/sequential_recommender/bert4rec.py +++ b/recbole/model/sequential_recommender/bert4rec.py @@ -82,6 +82,7 @@ def __init__(self, config, dataset): self.dropout = nn.Dropout(self.hidden_dropout_prob) self.output_ffn = nn.Linear(self.hidden_size, self.hidden_size) self.output_gelu = nn.GELU() + self.output_ln = nn.LayerNorm(self.hidden_size, eps=self.layer_norm_eps) self.output_bias = nn.Parameter(torch.zeros(self.n_items)) # we only need compute the loss at the masked position @@ -133,6 +134,7 @@ def forward(self, item_seq): input_emb, extended_attention_mask, output_all_encoded_layers=True ) ffn_output = self.output_ffn(trm_output[-1]) + ffn_output = self.output_ln(ffn_output) output = self.output_gelu(ffn_output) return output # [B L H] From 11394e940d68c1061afa26061d6ad30c1cfee9d0 Mon Sep 17 00:00:00 2001 From: David <1282675518@qq.com> Date: Wed, 13 Sep 2023 11:24:37 +0800 Subject: [PATCH 07/17] fix order --- recbole/model/sequential_recommender/bert4rec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recbole/model/sequential_recommender/bert4rec.py b/recbole/model/sequential_recommender/bert4rec.py index c936f67b6..936e57d77 100644 --- a/recbole/model/sequential_recommender/bert4rec.py +++ b/recbole/model/sequential_recommender/bert4rec.py @@ -134,8 +134,8 @@ def forward(self, item_seq): input_emb, extended_attention_mask, output_all_encoded_layers=True ) ffn_output = self.output_ffn(trm_output[-1]) - ffn_output = self.output_ln(ffn_output) - output = self.output_gelu(ffn_output) + ffn_output = self.output_gelu(ffn_output) + output = self.output_ln(ffn_output) return output # [B L H] def multi_hot_embed(self, masked_index, max_length): From 3b5ad8c357e20e5630d0f374175602ed4eb1d8b4 Mon Sep 17 00:00:00 2001 From: zbw <18735382001@163.com> Date: Tue, 19 Sep 2023 16:28:32 +0800 Subject: [PATCH 08/17] FIX: updata requirements.txt. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f92e2ec3a..ade8cf9fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ colorlog==4.7.2 colorama==0.4.4 tensorboard>=2.5.0 thop>=0.1.1.post2207130030 -ray>=1.13.0 +ray>=1.13.0, <=2.6.3 tabulate>=0.8.10 plotly>=4.0.0 texttable>=0.9.0 \ No newline at end of file From 53a5f0a37f063f4e87d37c1b25d3841cda476580 Mon Sep 17 00:00:00 2001 From: TayTroye <1582706091@qq.com> Date: Thu, 21 Sep 2023 19:39:24 +0800 Subject: [PATCH 09/17] fixed(trainer):change content --- recbole/trainer/trainer.py | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/recbole/trainer/trainer.py b/recbole/trainer/trainer.py index b5863b67d..890d93ff1 100644 --- a/recbole/trainer/trainer.py +++ b/recbole/trainer/trainer.py @@ -19,6 +19,7 @@ import os + from logging import getLogger from time import time @@ -256,6 +257,8 @@ def _train_epoch(self, train_data, epoch_idx, loss_func=None, show_progress=Fals losses.item() if total_loss is None else total_loss + losses.item() ) self._check_nan(loss) + if isinstance(loss, torch.Tensor): + loss = loss.to(self.device) #fixed scaler.scale(loss + sync_loss).backward() if self.clip_grad_norm: clip_grad_norm_(self.model.parameters(), **self.clip_grad_norm) @@ -1150,18 +1153,11 @@ def __init__(self, config, model): super(LightGBMTrainer, self).__init__(config, model) self.lgb = __import__("lightgbm") - self.boost_model = config["lgb_model"] - self.silent = config["lgb_silent"] # train params self.params = config["lgb_params"] self.num_boost_round = config["lgb_num_boost_round"] self.evals = () - self.early_stopping_rounds = config["lgb_early_stopping_rounds"] - self.evals_result = {} - self.verbose_eval = config["lgb_verbose_eval"] - self.learning_rates = config["lgb_learning_rates"] - self.callbacks = None self.deval_data = self.deval_label = None self.eval_pred = self.eval_true = None @@ -1174,7 +1170,7 @@ def _interaction_to_lib_datatype(self, dataloader): dataset(lgb.Dataset): Data in the form of 'lgb.Dataset'. """ data, label = self._interaction_to_sparse(dataloader) - return self.lgb.Dataset(data=data, label=label, silent=self.silent) + return self.lgb.Dataset(data=data, label=label) def _train_at_once(self, train_data, valid_data): r""" @@ -1187,16 +1183,7 @@ def _train_at_once(self, train_data, valid_data): self.dvalid = self._interaction_to_lib_datatype(valid_data) self.evals = [self.dtrain, self.dvalid] self.model = self.lgb.train( - self.params, - self.dtrain, - self.num_boost_round, - self.evals, - early_stopping_rounds=self.early_stopping_rounds, - evals_result=self.evals_result, - verbose_eval=self.verbose_eval, - learning_rates=self.learning_rates, - init_model=self.boost_model, - callbacks=self.callbacks, + self.params, self.dtrain, self.num_boost_round, self.evals ) self.model.save_model(self.temp_file) From a344685853354ad06bfc93f814d78eabe071ea23 Mon Sep 17 00:00:00 2001 From: TayTroye <1582706091@qq.com> Date: Thu, 21 Sep 2023 20:07:54 +0800 Subject: [PATCH 10/17] fixed(trainer):change content --- recbole/trainer/trainer.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/recbole/trainer/trainer.py b/recbole/trainer/trainer.py index 890d93ff1..f86ba3e55 100644 --- a/recbole/trainer/trainer.py +++ b/recbole/trainer/trainer.py @@ -19,7 +19,6 @@ import os - from logging import getLogger from time import time @@ -1153,11 +1152,18 @@ def __init__(self, config, model): super(LightGBMTrainer, self).__init__(config, model) self.lgb = __import__("lightgbm") + self.boost_model = config["lgb_model"] + self.silent = config["lgb_silent"] # train params self.params = config["lgb_params"] self.num_boost_round = config["lgb_num_boost_round"] self.evals = () + self.early_stopping_rounds = config["lgb_early_stopping_rounds"] + self.evals_result = {} + self.verbose_eval = config["lgb_verbose_eval"] + self.learning_rates = config["lgb_learning_rates"] + self.callbacks = None self.deval_data = self.deval_label = None self.eval_pred = self.eval_true = None @@ -1170,7 +1176,7 @@ def _interaction_to_lib_datatype(self, dataloader): dataset(lgb.Dataset): Data in the form of 'lgb.Dataset'. """ data, label = self._interaction_to_sparse(dataloader) - return self.lgb.Dataset(data=data, label=label) + return self.lgb.Dataset(data=data, label=label, silent=self.silent) def _train_at_once(self, train_data, valid_data): r""" @@ -1183,7 +1189,16 @@ def _train_at_once(self, train_data, valid_data): self.dvalid = self._interaction_to_lib_datatype(valid_data) self.evals = [self.dtrain, self.dvalid] self.model = self.lgb.train( - self.params, self.dtrain, self.num_boost_round, self.evals + self.params, + self.dtrain, + self.num_boost_round, + self.evals, + early_stopping_rounds=self.early_stopping_rounds, + evals_result=self.evals_result, + verbose_eval=self.verbose_eval, + learning_rates=self.learning_rates, + init_model=self.boost_model, + callbacks=self.callbacks, ) self.model.save_model(self.temp_file) From f8ca6437a9f48f67dfd3d9c4259440b4c030c14e Mon Sep 17 00:00:00 2001 From: TayTroye <1582706091@qq.com> Date: Thu, 21 Sep 2023 20:44:51 +0800 Subject: [PATCH 11/17] fixed(trainer):change content --- recbole/trainer/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recbole/trainer/trainer.py b/recbole/trainer/trainer.py index f86ba3e55..c9c57ba75 100644 --- a/recbole/trainer/trainer.py +++ b/recbole/trainer/trainer.py @@ -257,7 +257,7 @@ def _train_epoch(self, train_data, epoch_idx, loss_func=None, show_progress=Fals ) self._check_nan(loss) if isinstance(loss, torch.Tensor): - loss = loss.to(self.device) #fixed + loss = loss.to(self.device) scaler.scale(loss + sync_loss).backward() if self.clip_grad_norm: clip_grad_norm_(self.model.parameters(), **self.clip_grad_norm) From af0597d3c19021c108399d4da706fc5d8912778e Mon Sep 17 00:00:00 2001 From: TayTroye <1582706091@qq.com> Date: Thu, 21 Sep 2023 21:04:13 +0800 Subject: [PATCH 12/17] FIX(Pop):change content --- recbole/model/general_recommender/pop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recbole/model/general_recommender/pop.py b/recbole/model/general_recommender/pop.py index 2984d23c6..829888515 100644 --- a/recbole/model/general_recommender/pop.py +++ b/recbole/model/general_recommender/pop.py @@ -43,7 +43,7 @@ def calculate_loss(self, interaction): self.max_cnt = torch.max(self.item_cnt, dim=0)[0] - return torch.nn.Parameter(torch.zeros(1)) + return torch.nn.Parameter(torch.zeros(1)).to(self.device) def predict(self, interaction): item = interaction[self.ITEM_ID] From 8fb6d3b1fd6aa4e1675581c1fef2d335f1b2cdad Mon Sep 17 00:00:00 2001 From: TayTroye <1582706091@qq.com> Date: Thu, 21 Sep 2023 21:21:12 +0800 Subject: [PATCH 13/17] FIX(POP):add content --- recbole/model/general_recommender/pop.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/recbole/model/general_recommender/pop.py b/recbole/model/general_recommender/pop.py index 829888515..a4d75d39c 100644 --- a/recbole/model/general_recommender/pop.py +++ b/recbole/model/general_recommender/pop.py @@ -6,6 +6,10 @@ # @Time : 2020/11/9 # @Author : Zihan Lin # @Email : zhlin@ruc.edu.cn +# UPDATE +# @Time :2023/9/21 +# @Author : Kesha Ou +# @Email :1582706091@qq.com r""" Pop From b66f079ad6b41c22e768751654c5459389e04406 Mon Sep 17 00:00:00 2001 From: TayTroye <1582706091@qq.com> Date: Thu, 21 Sep 2023 21:22:57 +0800 Subject: [PATCH 14/17] reset --- recbole/trainer/trainer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/recbole/trainer/trainer.py b/recbole/trainer/trainer.py index c9c57ba75..b5863b67d 100644 --- a/recbole/trainer/trainer.py +++ b/recbole/trainer/trainer.py @@ -256,8 +256,6 @@ def _train_epoch(self, train_data, epoch_idx, loss_func=None, show_progress=Fals losses.item() if total_loss is None else total_loss + losses.item() ) self._check_nan(loss) - if isinstance(loss, torch.Tensor): - loss = loss.to(self.device) scaler.scale(loss + sync_loss).backward() if self.clip_grad_norm: clip_grad_norm_(self.model.parameters(), **self.clip_grad_norm) From dbaeb5f19d5facaf29b2fcdfa7b561d48fa56db1 Mon Sep 17 00:00:00 2001 From: zbw <18735382001@163.com> Date: Sat, 23 Sep 2023 15:54:43 +0800 Subject: [PATCH 15/17] FIX: fix the bug in negative sampling. --- recbole/data/dataloader/abstract_dataloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recbole/data/dataloader/abstract_dataloader.py b/recbole/data/dataloader/abstract_dataloader.py index 5bc630b94..afa39a213 100644 --- a/recbole/data/dataloader/abstract_dataloader.py +++ b/recbole/data/dataloader/abstract_dataloader.py @@ -137,7 +137,7 @@ def _set_neg_sample_args(self, config, dataset, dl_format, neg_sample_args): self.times = 1 if ( self.neg_sample_args["distribution"] == "uniform" - or "popularity" + or self.neg_sample_args["distribution"] == "popularity" and self.neg_sample_args["sample_num"] != "none" ): self.neg_sample_num = self.neg_sample_args["sample_num"] From c5be7efd49cf4d5300848bd75209cc440b626d6c Mon Sep 17 00:00:00 2001 From: Bowen Zheng <57226275+zhengbw0324@users.noreply.github.com> Date: Tue, 26 Sep 2023 08:53:13 +0800 Subject: [PATCH 16/17] Update abstract_dataloader.py --- recbole/data/dataloader/abstract_dataloader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recbole/data/dataloader/abstract_dataloader.py b/recbole/data/dataloader/abstract_dataloader.py index afa39a213..59f6398fd 100644 --- a/recbole/data/dataloader/abstract_dataloader.py +++ b/recbole/data/dataloader/abstract_dataloader.py @@ -136,8 +136,8 @@ def _set_neg_sample_args(self, config, dataset, dl_format, neg_sample_args): self.neg_sample_args = neg_sample_args self.times = 1 if ( - self.neg_sample_args["distribution"] == "uniform" - or self.neg_sample_args["distribution"] == "popularity" + (self.neg_sample_args["distribution"] == "uniform" + or self.neg_sample_args["distribution"] == "popularity") and self.neg_sample_args["sample_num"] != "none" ): self.neg_sample_num = self.neg_sample_args["sample_num"] From 6d9f74216379ccfac068be9769cb3a2858d4fd14 Mon Sep 17 00:00:00 2001 From: zhengbw0324 Date: Tue, 26 Sep 2023 01:10:42 +0000 Subject: [PATCH 17/17] Format Python code according to PEP8 --- recbole/data/dataloader/abstract_dataloader.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/recbole/data/dataloader/abstract_dataloader.py b/recbole/data/dataloader/abstract_dataloader.py index 59f6398fd..b97b33174 100644 --- a/recbole/data/dataloader/abstract_dataloader.py +++ b/recbole/data/dataloader/abstract_dataloader.py @@ -136,10 +136,9 @@ def _set_neg_sample_args(self, config, dataset, dl_format, neg_sample_args): self.neg_sample_args = neg_sample_args self.times = 1 if ( - (self.neg_sample_args["distribution"] == "uniform" - or self.neg_sample_args["distribution"] == "popularity") - and self.neg_sample_args["sample_num"] != "none" - ): + self.neg_sample_args["distribution"] == "uniform" + or self.neg_sample_args["distribution"] == "popularity" + ) and self.neg_sample_args["sample_num"] != "none": self.neg_sample_num = self.neg_sample_args["sample_num"] if self.dl_format == InputType.POINTWISE: