This repository has been archived by the owner on Jan 7, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
/
excavator.js
133 lines (118 loc) · 3.32 KB
/
excavator.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
#!/usr/bin/env node
const fs = require('fs');
const parser = require('parse-rss');
const get = require('lodash/get');
const sortBy = require('lodash/sortBy');
const reverse = require('lodash/reverse');
const cities = require('./src/cities');
const execFileSync = require('child_process').execFileSync;
const prompt = '🚜 🚜 🚜 Dirt Excavator 3000';
function exec(command, args, hideFunction) {
if (!hideFunction) {
console.log(`🚜 > ${[command].concat(args).join(' ')}`);
}
const options = {};
return execFileSync(command, args, options).toString();
}
function githubSetup() {
exec('git', ['config', '--global', 'user.name', '🚜 🤖']);
exec('git', [
'config',
'--global',
'user.email',
]);
exec('git', ['init']);
exec(
'git',
[
'remote',
'add',
'origin',
'https://' +
process.env.GITHUB_USER +
':' +
process.env.GITHUB_TOKEN +
'@github.com/chrisheninger/freedirtalert.com.git',
],
true
);
}
function shovel(city, cb) {
console.log(`🚜 > Let's shovel some dirt in ${cities[city]}! 🚨 🌎 🆓`);
// Parse RSS feed to JSON
parser(
`https://${city}.craigslist.org/search/zip?format=rss&query=Dirt`,
function(err, res) {
if (err) {
console.error(city, err);
throw err;
}
let dirts = [];
// Take response and create a new object for each listing with all the data we will need
res.forEach(item => {
const listing = {
title:
get(item, 'title') && get(item, 'title').split(/\(([^)]+)\)/)[0],
location:
(get(item, 'title') &&
get(item, 'title').split(/\(([^)]+)\)/)[1]) ||
'---',
description: get(item, 'description'),
date: get(item, 'date'),
link: get(item, 'link'),
image:
(get(item, ['enc:enclosure', '@', 'resource']) &&
get(item, ['enc:enclosure', '@', 'resource']).replace(
'300x300.jpg',
'600x450.jpg'
)) ||
'https://freedirtalert.com/images/no-dirt.png',
};
dirts.push(listing);
});
// Sort by date then reverse order so newest are on top
dirts = sortBy(dirts, listing => {
return new Date(listing.date);
});
dirts = reverse(dirts);
const jsonResults = {
city,
date: Date.now(),
data: dirts,
};
// Save json file to disk in data directory
fs.writeFile(
`./public/data/${city}.json`,
JSON.stringify(jsonResults),
'utf8',
function(err) {
if (err) throw err;
console.log(
`🚜 > All the dirt in ${cities[city]} has been shoveled! 🚨 🌎 🆓`
);
cb();
}
);
}
);
}
/**
* The script
*/
function loop() {
console.log(`${prompt}: Time to dig!`);
const requests = Object.keys(cities).map(city => {
return new Promise(resolve => {
shovel(city, resolve);
});
});
Promise.all(requests).then(() => {
console.log(`${prompt}: Finished digging!`);
exec('yarn', ['run', 'deploy']);
console.log('That was exhausting... taking a one hour break. 🚧 👋 ⏰');
setTimeout(loop, 3600000);
});
}
githubSetup();
loop();