fixed bug in api writer

jakopako · Apr 30, 2022 · 6cb3013 · 6cb3013
1 parent 789de04
commit 6cb3013
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 14 deletions.
diff --git a/concerts-config.yml b/concerts-config.yml
@@ -4,7 +4,9 @@
 
 writer:
   type: "api"
-  # The other parameters are defined as env variables.
+  uri: "http://localhost:5000/api/events"
+  user: "croncert"
+  password: "croncert"
 
 # Scraper configs are grouped by city and cities are sorted alphabetically.
 scrapers:
@@ -379,8 +381,6 @@ scrapers:
           value: "concert"
         - name: "city"
           value: "Munich"
-        - name: "location"
-          value: "Muffatwerk"
       dynamic:
         - name: "title"
           location:
@@ -390,6 +390,14 @@ scrapers:
           location:
             selector: ".hover-in .right a"
           relative: true
+        - name: "location"
+          on_subpage: "url"
+          location:
+            selector: ".entry-info"
+            child_index: 10
+            regex_extract:
+              exp: "[^:]+"
+              index: 1
         - name: "date"
           type: "date"
           components:
@@ -419,6 +427,19 @@ scrapers:
                   exp: "[0-9]{2}:[0-9]{2}"
               layout: ["15:04"]
           date_location: "Europe/Berlin"
+    filters:
+      - field: "location"
+        regex: "Zenith" # duplicate (also present on Motorworld website)
+        match: false
+      - field: "location"
+        regex: "Strom" # duplicate
+        match: false
+      - field: "location"
+        regex: "Tonhalle" # duplicate
+        match: false
+      - field: "location"
+        regex: "TonHalle" # duplicate
+        match: false
 
   ##########
   # Paris

diff --git a/output/api.go b/output/api.go
@@ -36,22 +36,29 @@ func (f *APIWriter) Write(itemsList chan []map[string]interface{}) {
 	for items := range itemsList {
 		if len(items) > 0 {
 			// delete events of this scraper from first date on
-			firstDate := items[0]["date"].(time.Time).UTC().Format("2006-01-02 15:04")
-			deleteURL := fmt.Sprintf("%s?location=%s&datetime=%s", apiURL, url.QueryEscape(items[0]["location"].(string)), url.QueryEscape(firstDate))
-			req, _ := http.NewRequest("DELETE", deleteURL, nil)
-			req.SetBasicAuth(apiUser, apiPassword)
-			resp, err := client.Do(req)
-			if err != nil {
-				log.Fatal(err)
+			// a list of events might contain multiple events
+			locations := map[string]bool{}
+			for _, item := range items {
+				locations[item["location"].(string)] = true
 			}
-			if resp.StatusCode != 200 {
-				body, err := ioutil.ReadAll(resp.Body)
+			firstDate := items[0]["date"].(time.Time).UTC().Format("2006-01-02 15:04")
+			for loc := range locations {
+				deleteURL := fmt.Sprintf("%s?location=%s&datetime=%s", apiURL, url.QueryEscape(loc), url.QueryEscape(firstDate))
+				req, _ := http.NewRequest("DELETE", deleteURL, nil)
+				req.SetBasicAuth(apiUser, apiPassword)
+				resp, err := client.Do(req)
 				if err != nil {
 					log.Fatal(err)
 				}
-				log.Fatalf("something went wrong while deleting events. Status Code: %d\nUrl: %s Response: %s", resp.StatusCode, deleteURL, body)
+				if resp.StatusCode != 200 {
+					body, err := ioutil.ReadAll(resp.Body)
+					if err != nil {
+						log.Fatal(err)
+					}
+					log.Fatalf("something went wrong while deleting events. Status Code: %d\nUrl: %s Response: %s", resp.StatusCode, deleteURL, body)
+				}
+				resp.Body.Close()
 			}
-			resp.Body.Close()
 			// add new events
 			for _, item := range items {
 				concertJSON, err := json.Marshal(item)