added_multiprocessing

Notifier-service/logs/App.log

@@ -0,0 +1,23 @@
+{"@message":"Application listening at port :3000","@fields":{"level":"info"}}
+{"@message":"Application listening at port :3000","@fields":{"level":"info"}}
+{"message":"Application listening at port :3000","level":"info"}
+undefined info: "Application listening at port :3000"
+undefined info: "Application listening at port :3000"
+{"message":"Application listening at port :3000","level":"info"}
+{"message":"Application listening at port :3000","level":"info"}
+{"message":"Application listening at port :3000","level":"info"}
+{"message":"Application listening at port :3000","level":"info"}
+undefined
+2020-03-09T15:54:59.751Z - App:[info]: >>>>>>>>>>>>>>>>>>>>>>>>>>>> APPLICATION STARTED <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+2020-03-09T15:54:59.754Z - App:[info]: Application listening at port :3000
+2020-03-09T15:58:42.804Z - App:[info]: >>>>>>>>>>>>>>>>>>>>>>>>>>>> APPLICATION STARTED <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+2020-03-09T15:58:42.807Z - App:[info]: Application listening at port :3000
+2020-03-09T15:59:07.249Z - App:[info]: >>>>>>>>>>>>>>>>>>>>>>>>>>>> APPLICATION STARTED <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+2020-03-09T15:59:07.252Z - App:[info]: Application listening at port :3000
+2020-03-09T16:00:18.749Z - App:[info]: >>>>>>>>>>>>>>>>>>>>>>>>>>>> APPLICATION STARTED <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+2020-03-09T16:00:18.752Z - App:[info]: Application listening at port :3000
+2020-03-09T16:01:01.375Z - App:[info]: >>>>>>>>>>>>>>>>>>>>>>>>>>>> APPLICATION STARTED <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+2020-03-09T16:01:01.378Z - App:[info]: Application listening at port :3000
+2020-03-09T16:01:01.380Z - App:[warn]: Thi is warning
+2020-03-09T16:01:57.159Z - App:[info]: >>>>>>>>>>>>>>>>>>>>>>>>>>>> APPLICATION STARTED <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+2020-03-09T16:01:57.163Z - App:[info]: Application listening at port :3000

Notifier-service/src/app.ts

@@ -5,9 +5,14 @@ import { Database} from './interfaces/database';
 import { Server } from 'http';
 import { Controller } from './interfaces/controller';
 import { Model } from './interfaces/model';
+import { Logger } from './utility/log';
 
 
 export class App {
+
+    private logger = new Logger(this.constructor.name).getLogger();
+
+
     private app:Application;
     private server:Server|undefined;
     constructor(private routes:Array<Route>,
@@ -50,6 +55,8 @@ export class App {
             throw new Error("No proper port for server found , configure in .env file")
         let port = Number.parseInt(process.env.SERVER_PORT)
         this.server = this.app.listen( port , () => {
+            this.logger.info(">>>>>>>>>>>>>>>>>>>>>>>>>>>> APPLICATION STARTED <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<");
+            this.logger.info("Application listening at port :"+ port);
             callback(port);
         })
     }

Notifier-service/src/controllers/conference.ts

@@ -13,14 +13,26 @@ export class ConferenceController extends Controller {
 
     async getConferences(offset:Number , count:Number):Promise<Conference[]>{
         // TO-DO
+        /*
+            This route will return count number of Conferenes data , with a particular offset
+            The results should be sorted by nearest date greater than current date
+        */
         return Promise.resolve([]);
     }
     async getConferencesFromCategory(category:String , offset:Number , count:Number):Promise<Conference[]>{
         //TO- DO
+        /*
+            This route will return count number of Conferenes data , with a particular offset
+            The results should be sorted by nearest date greater than current date
+            But also the result should be of a a particular conference.
+        */
         return Promise.resolve([]);
     }
     async getCategories():Promise<Array<String>>{
         // To-DO
+        /*
+            get all the conference category
+        */
         return Promise.resolve([]);
     };
 

Notifier-service/src/models/conference.ts

@@ -30,6 +30,20 @@ export class ConferenceModel extends Model{
         })
         return result
     }
+
+
+    /*
+        TO-DO 
+        1. add getConferences query function for the controller
+            -> Will get range number of conferences with a given offset
+            -> the result will be sorted by deadline
+
+        2. add getConferencesFromCategory query function for the controller
+            -> same as above but will have extra parameter of conference category
+
+        3. add getCategories query function for the controller
+            -> returns all unique categories acrooss all the entries
+    */
 }
 
 

Notifier-service/src/utility/log.ts

@@ -0,0 +1,59 @@
+import winston from 'winston';
+
+// const FileConfig = (filename:string) =>{ 
+//     return {
+//             level: process.env.LOG_LEVEL,
+//             filename: `${process.env.LOG_FOLDER}/${filename}.log`,
+//             handleExceptions: true,
+//             json: true,
+//             colorize: false,
+//             timestamp:true
+//         } 
+// }
+
+// const ConsoleConfig = {
+//     level: process.env.LOG_LEVEL,
+//     handleExceptions: true,
+//     colorize: true,
+//     prettyPrint: true
+// }
+
+
+export class Logger{
+
+
+    private logger:winston.Logger; 
+    constructor(filename:string){
+        this.logger = winston.createLogger({
+            level: process.env.LOG_LEVEL,
+            transports: [
+                new winston.transports.Console({ format: winston.format.colorize({all:true}),}),
+                new winston.transports.File({ filename: `${process.env.LOG_FOLDER}/${filename}.log` })
+            ] , 
+
+            format: winston.format.combine(
+                         winston.format.label({
+                            label: filename
+                         }),
+
+                         winston.format.timestamp(),
+                         winston.format.printf((info) => {
+                             return `${info.timestamp} - ${info.label}:[${info.level}]: ${info.message}`;
+                         })
+                     )
+
+        })
+    }    
+
+    getLogger():winston.Logger{
+        return this.logger;
+    }
+
+}
+
+
+
+
+
+
+

Scrapper-Service/app.py

@@ -5,70 +5,112 @@
 import traceback
 import os
 from utility import str2bool , get_logger , print_start
-from process import Multiprocessing
-
+from process import MultiProcessingContext
 
 
 ##
 #
 # Test whether the initialization is working or not
 # >> python app.py -l debug -ls console  --test True
-# 
+#
 # Test whether the run is working or not
 # >> python app.py -l debug -ls file --test False
-#  
-# 
+#
+#
 
 parser = argparse.ArgumentParser()
-parser.add_argument("-c" , "--config",  default='config.json' , type=str , action='store'
-                                     ,  dest="config" , help="Specify config.json file")
-parser.add_argument("-l" , "--log" ,    default="debug"  , action='store', dest="log_level" ,
-                                        choices=["debug" , "warn" , "error" , "info"]  , help="Specify the debug level ,default: %(default)s")
-parser.add_argument("-t" , "--test" ,   default=True , type=str2bool , action="store" , dest="test" , 
-                                        help="Specify whether to test app initialization or run the scrappers ,default: %(default)s")
-parser.add_argument("-ls", "--logStream", default="console", type=str , action="store" , dest="log_stream" , 
-                                          choices=["console" , "file"],
-                                          help="Specify whether to print logs on terminal or to file ,default: %(default)s"  )
+parser.add_argument(
+    "-c",
+    "--config",
+    default="config.json",
+    type=str,
+    action="store",
+    dest="config",
+    help="Specify config.json file",
+)
+parser.add_argument(
+    "-l",
+    "--log",
+    default="debug",
+    action="store",
+    dest="log_level",
+    choices=["debug", "warn", "error", "info"],
+    help="Specify the debug level ,default: %(default)s",
+)
+parser.add_argument(
+    "-t",
+    "--test",
+    default=True,
+    type=str2bool,
+    action="store",
+    dest="test",
+    help="Specify whether to test app initialization or run the scrappers ,default: %(default)s",
+)
+parser.add_argument(
+    "-ls",
+    "--logStream",
+    default="console",
+    type=str,
+    action="store",
+    dest="log_stream",
+    choices=["console", "file"],
+    help="Specify whether to print logs on terminal or to file ,default: %(default)s",
+)
 values = parser.parse_args()
 
-LOG_LEVEL_DEFAULTS = {"debug":logging.DEBUG , "warn":logging.WARN , "info":logging.INFO , "error":logging.ERROR}
-LOG_STREAM_DEFAULTS = { "file":lambda filename:logging.FileHandler(filename),
-                        "console":lambda filname: logging.StreamHandler()
-                        }
+LOG_LEVEL_DEFAULTS = {
+    "debug": logging.DEBUG,
+    "warn": logging.WARN,
+    "info": logging.INFO,
+    "error": logging.ERROR,
+}
+LOG_STREAM_DEFAULTS = {
+    "file": lambda filename: logging.FileHandler(filename),
+    "console": lambda filename: logging.StreamHandler(),
+}
 
 if values.log_level.lower() not in LOG_LEVEL_DEFAULTS:
-    raise ValueError("Unsupported log level. Supported levels: debug , warn , info , error")
+    raise ValueError(
+        "Unsupported log level. Supported levels: debug , warn , info , error"
+    )
 if values.log_stream.lower() not in LOG_STREAM_DEFAULTS:
     raise ValueError("Unsupported log stream. Supported levels: file , console")
 
 log_level = LOG_LEVEL_DEFAULTS[values.log_level.lower()]
 
-istest = values.test
+is_test = values.test
 
 CONFIG = values.config
 
 log_streamOption = LOG_STREAM_DEFAULTS[values.log_stream.lower()]
-
 
-def createDatabase(configuration):
+
+def parse_dbconfig(configuration):
     db_configuration = configuration["database"]
     path = db_configuration["plugin"]["filename"]
     classname = db_configuration["plugin"]["class"]
-    module = importlib.import_module(path , ".")
+    module = importlib.import_module(path, ".")
     Database = module.__getattribute__(classname)
-    return lambda logger: Database(logger , **db_configuration) 
+    return Database , db_configuration
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
 
     with open(CONFIG) as file:
         configuration = json.load(file)
 
+    Database_module  , db_configuration = parse_dbconfig(configuration)
+
     ## reading logging configuration
     logging_configuration = configuration["logging"]
     log_folder = logging_configuration["output"]
     if not log_folder in os.listdir('.'):
         os.mkdir(log_folder)
 
+    logger = get_logger(
+        __name__, log_level, log_streamOption("{}/{}.log".format(log_folder, "main"))
+    )
+
     ## logger for main thread
     logger = get_logger(__name__ , log_level ,  log_streamOption("{}/{}.log".format(log_folder , "main")) )
 
@@ -79,30 +121,33 @@ def createDatabase(configuration):
 
     ## handles creating mutiple process 
     ## from single process using MultiProcessing  
-    multip = Multiprocessing()
+
 
     import_list = configuration["plugins"]
-    for attr in import_list:
-
-        path = attr["filename"]
-        class_name = attr["class"]
-        plugin_module = importlib.import_module(path , ".")
-        scrapper = plugin_module.__getattribute__(class_name)
-        try:
-            log_stream = log_streamOption("{}/{}.log".format(log_folder , class_name))
-            if istest:
-                multip.execute_process( 
-                    lambda : scrapper(  log_level = log_level, 
-                                        log_stream = log_stream , 
-                                        getDatabaseObject = createDatabase(configuration) 
-                                    ))
-            else:
-                multip.execute_process(
-                    lambda : scrapper(  log_level = log_level, 
-                                        log_stream = log_stream , 
-                                        getDatabaseObject = createDatabase(configuration) ).run() )
-
-        except Exception as e:
-            logger.error("{} scrapper failed".format(class_name))
-            traceback.print_exception(type(e), e, e.__traceback__)
-    logger.info("Scrapping done from all Scrapper plugins")
+
+    with MultiProcessingContext( log_level , log_streamOption("{}/{}.log".format(log_folder , "multiproc")) ) as execute:
+
+        for attr in import_list:
+            path = attr["filename"]
+            class_name = attr["class"]
+            plugin_module = importlib.import_module(path, ".")
+            scrapper = plugin_module.__getattribute__(class_name)
+            try:
+                log_stream = log_streamOption("{}/{}.log".format(log_folder , class_name))
+                if is_test:
+                    scrapper(  log_level = log_level, 
+                                            log_stream = log_stream , 
+                                            database_module = Database_module, 
+                                            db_configuration = db_configuration
+                                        )
+                else:
+                    execute(    scrapper(  log_level = log_level, 
+                                            log_stream = log_stream ,
+                                            database_module = Database_module, 
+                                            db_configuration = db_configuration
+                                            ).run )
+
+            except Exception as e:
+                logger.error("{} scrapper failed".format(class_name))
+                traceback.print_exception(type(e), e, e.__traceback__)
+    logger.info("Scrapping done from all Scrapper plugins")

Scrapper-Service/commons/scrapper.py

@@ -27,10 +27,15 @@ def get_date(self , string , fmt = "%b %d, %Y" ):
             return string
 
 
-    def __init__(self , context_name , log_level , log_stream  , getDatabaseObject = lambda logger: None ,  **kwargs):
+
+
+    def __init__(self , context_name , log_level , log_stream  , database_module , db_configuration ,   **kwargs):
         self.logger = get_logger(context_name , log_level , log_stream)  
         print_start(self.logger)   
-        self.db = getDatabaseObject(self.logger)
+
+        self.db = database_module(self.logger , **db_configuration)
+
+
         self.logger.info("{} setup complete !!".format(context_name))
         if self.db != None:
             self.push_todb = self.db.put

Scrapper-Service/process/__init__.py

@@ -1 +1 @@
-from .mutiprocessing import Multiprocessing
+from .mutiprocessing import MultiProcessingContext