Merge pull request #8 from beclab/feat/train-rank-replace-mongo-with-pg

feat: train rank replace mongo with pg

prerank-stages/api/knowledge.go

@@ -69,10 +69,12 @@ func AddEntriesInMongo(list []*model.EntryModel) {
 		if resObj.Code == 0 {
 			resEntryMap := make(map[string]string, 0)
 			for _, resDataDetail := range resObj.Data {
+				common.Logger.Info("response entry direct",zap.String("url",resDataDetail.Url),zap.String("id",resDataDetail.ID),zap.String("source",resDataDetail.Source))
 				resEntryMap[resDataDetail.Url] = resDataDetail.ID
 			}
 			addAlgorithmList := make([]*model.AlgorithmAddModel, 0)
 			for _, entryModel := range list {
+				common.Logger.Info("construct add algorithm",zap.String("url",entryModel.Url),zap.String("id", resEntryMap[entryModel.Url]))
 				algoModel := model.GetAddAlgorithmModel(resEntryMap[entryModel.Url], entryModel.RecallPoint, entryModel.PrerankPoint, entryModel.Embedding)
 				addAlgorithmList = append(addAlgorithmList, algoModel)
 			}
@@ -87,6 +89,9 @@ func AddEntriesInMongo(list []*model.EntryModel) {
 
 func UpdateEntryAlgorith(addAlgorithmList []*model.AlgorithmAddModel) {
 	if len(addAlgorithmList) > 0 {
+		for _,currentAlgorithm := range addAlgorithmList {
+			common.Logger.Info("current_algorithm",zap.String("source",currentAlgorithm.Source),zap.String("entry_id",currentAlgorithm.Entry))
+		}
 		algoUrl := common.AlgorithMonogoApiUrl()
 		algoJsonByte, err := json.Marshal(addAlgorithmList)
 		if err != nil {

prerank-stages/dev_container/Dockerfile

@@ -0,0 +1,26 @@
+FROM ubuntu:jammy
+
+RUN apt-get update && \
+    apt-get install build-essential -y && \
+    apt-get install cmake -y && \
+    apt install git -y && \
+    apt install curl -y && \
+    apt install pkg-config -y && \
+    apt-get install libssl-dev -y && \
+    apt-get install unzip -y && \
+    apt-get install protobuf-compiler -y
+
+RUN  cd /opt && \
+     curl -LO https://go.dev/dl/go1.20.11.linux-amd64.tar.gz && \
+     tar -xvzf go1.20.11.linux-amd64.tar.gz && \
+     mv go /usr/local 
+ENV PATH $PATH:/usr/local/go/bin
+RUN go env -w GO111MODULE=on
+
+RUN mkdir -p /opt/prerank-stages && \
+    go install google.golang.org/protobuf/cmd/protoc-gen-go@latest  
+
+
+
+
+ENTRYPOINT ["tail", "-f", "/dev/null"]

prerank-stages/dev_container/build.sh

@@ -0,0 +1,10 @@
+dev_container_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 
+echo $dev_container_dir
+root_dir=$(dirname -- "$dev_container_dir") 
+echo $root_dir
+DOCKER_FILE_PATH=$dev_container_dir/Dockerfile
+PREFIX=beclab
+
+docker  build    \
+    -f ${DOCKER_FILE_PATH} \
+    -t ${PREFIX}/prerank_stages_develop $root_dir

prerank-stages/dev_container/readme.md

@@ -0,0 +1,2 @@
+export host_code_directory="/home/ubuntu/r4/prerank-stages"
+docker run --name prerank_stage_develop  -v $host_code_directory:/opt/prerank-stages --net=host -d beclab/prerank_stages_develop

prerank-stages/model/entry.go

@@ -52,7 +52,7 @@ type EntryDelModel struct {
 }
 
 type EntryAddResponseModel struct {
-	ID     string `json:"_id,omitempty" `
+	ID     string `json:"id,omitempty" `
 	Source string `json:"source"`
 	Url    string `json:"url"`
 }

train-rank/src/entity/entry.h

@@ -4,7 +4,7 @@
 #include <string>
 #include <vector>
 
-static const char ENTRY_ID[] = "_id";
+static const char ENTRY_ID[] = "id";
 static const char ENTRY_FILE_TYPE[] = "file_type";
 static const char ENTRY_READ_LATER[] = "readlater";
 static const char ENTRY_CRAWLER[] = "crawler";

train-rank/src/entity/impression.h

@@ -9,7 +9,7 @@ static const char IMPRESSION_MONGO_FIELD_BATCH_ID[] = "batch_id";
 static const char IMPRESSION_MONGO_FIELD_POSITION[] = "position";
 static const char IMPRESSION_MONGO_FIELD_READ_TIME[] = "read_time";
 static const char IMPRESSION_MONGO_FIELD_READ_FINISH[] = "read_finish";
-static const char IMPRESSION_MONGO_FIELD_ID[] = "_id";
+static const char IMPRESSION_MONGO_FIELD_ID[] = "id";
 static const char IMPRESSION_MONGO_FIELD_ENTRY_ID[] = "entry_id";
 static const char IMPRESSION_MONGO_FIELD_CLICKED[] = "clicked";
 static const char IMPRESSION_MONGO_FIELD_ALGORITHM_EXTRA[] = "algorithm_extra";

train-rank/src/entity/rank_algorithm.h

@@ -4,7 +4,7 @@
 #include <vector>
 #include <optional>
 
-static const char ALGORITHM_MONGO_FIELD_ID[] = "_id";
+static const char ALGORITHM_MONGO_FIELD_ID[] = "id";
 static const char ALGORITHM_MONGO_FIELD_ENTRY[] = "entry";
 static const char ALGORITHM_MONGO_FIELD_SOURCE[] = "source";
 static const char ALGORITHM_MONGO_FIELD_SCORE[] = "score";

user-embedding/README.md

@@ -15,8 +15,8 @@ bash build.sh
 
 # Develop Environment
 ```bash
-host_code_directory="/home/ubuntu/recommend-bytetrade-algorithm/user-embedding"
-host_hugging_face_directory="/home/ubuntu/huggingface"
+export host_code_directory="/home/ubuntu/r4/user-embedding"
+export host_hugging_face_directory="/home/ubuntu/huggingface"
 docker run --name temp_userembedding_develop -v $host_code_directory:/opt/rss-termius-v2-userembedding -v $host_hugging_face_directory:/root/.cache/huggingface --net=host -d beclab/userembedding_develop
 ```
 Use VSCode to open the user-embedding directory, then attach to the `temp_userembedding_develop` container.

user-embedding/src/entity/entry.rs

@@ -1,4 +1,4 @@
-pub const MONGO_ID: &str = "_id";
+pub const MONGO_ID: &str = "id";
 pub const MONGO_FEED_ID: &str = "feed";
 pub const MONGO_STATUS: &str = "status";
 pub const MONGO_TITLE: &str = "title";

user-embedding/src/entity/impression.rs

@@ -1,4 +1,4 @@
-pub const MONGO_ID: &str = "_id";
+pub const MONGO_ID: &str = "id";
 pub const MONGO_ENTRY_ID: &str = "entry_id";
 pub const MONGO_SOURCE: &str = "source";
 pub const MONGO_BATCH_ID: &str = "batch_id";