Skip to content

Commit

Permalink
fix some minor suggestions and warnings and deprecations
Browse files Browse the repository at this point in the history
  • Loading branch information
Yolgie committed Oct 7, 2023
1 parent 44ea72d commit f2eaead
Show file tree
Hide file tree
Showing 27 changed files with 142 additions and 140 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ class EventCollectorWebUi(port: Int, private val scheduler: EventCollectorSchedu
}

private fun setupIndex() {
server.createContext("/") {
server.createContext("/") { httpExchange ->
try {
val context = VelocityContext()

Expand All @@ -62,18 +62,18 @@ class EventCollectorWebUi(port: Int, private val scheduler: EventCollectorSchedu
context.put("fullCollections",
Collections.getAllPastCollections().map { mapFullCollectionToFrontEnd(it) })

sendResponse(it, "/html/index.html.vm", context)
sendResponse(httpExchange, "/html/index.html.vm", context)
} catch (e: Exception) {
LOG.error("error while handling request", e)
send500(it)
send500(httpExchange)
}
}
}

private fun setupSingleCollection() {
server.createContext("/singleCollection") {
server.createContext("/singleCollection") { httpExchange ->
try {
val id = parseId(it.requestURI.query)
val id = parseId(httpExchange.requestURI.query)
if (id != null) {
val singleCollection = findSingleCollection(id)
if (singleCollection != null) {
Expand All @@ -90,14 +90,14 @@ class EventCollectorWebUi(port: Int, private val scheduler: EventCollectorSchedu
)
context.put("log", formatLogLines(singleCollection.logLines))

sendResponse(it, "/html/singleCollection.html.vm", context)
sendResponse(httpExchange, "/html/singleCollection.html.vm", context)
return@createContext
}
}
send404(it)
send404(httpExchange)
} catch (e: Exception) {
e.printStackTrace()
send500(it)
send500(httpExchange)
}
}
}
Expand Down Expand Up @@ -194,7 +194,7 @@ class EventCollectorWebUi(port: Int, private val scheduler: EventCollectorSchedu

private fun findSingleCollection(id: UUID): SingleCollection? {
return Collections.getAllPastCollections()
.union(listOf(Collections.getCurrentFullCollection()).filterNotNull())
.union(listOfNotNull(Collections.getCurrentFullCollection()))
.flatMap { it.singleCollections }
.find { it.id == id }
}
Expand Down Expand Up @@ -256,7 +256,7 @@ class EventCollectorWebUi(port: Int, private val scheduler: EventCollectorSchedu
}

private fun formatLogLines(logLines: List<Pair<Boolean, ByteArray>>): String? =
EscapeTool().html(logLines.map { String(it.second) }.joinToString("\n"))
EscapeTool().html(logLines.joinToString("\n") { String(it.second) })

private fun setupStaticFolder(prefix: String) {
server.createContext(prefix) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,14 @@ import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpRequest.BodyPublishers
import java.net.http.HttpResponse.BodyHandlers
import kotlin.math.max

/* how long we want to wait between request as to not overload the target server */
const val DEFAULT_MIN_WAIT_TIME_IN_MS = 100L

class Fetcher(waitTimeInMs: Long = DEFAULT_MIN_WAIT_TIME_IN_MS) {

private val realWaitTimeInMs = Math.max(DEFAULT_MIN_WAIT_TIME_IN_MS, waitTimeInMs)
private val realWaitTimeInMs = max(DEFAULT_MIN_WAIT_TIME_IN_MS, waitTimeInMs)
private val newHttpClient = HttpClient.newHttpClient()

private var lastRequest = 0L
Expand Down
1 change: 1 addition & 0 deletions eventcollectors/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ dependencies {
exclude("org.codehaus.groovy", "groovy-dateutil")
}
implementation("com.rometools:rome:2.1.0")
implementation("org.apache.httpcomponents:httpclient:4.5.13")
implementation("com.beust:klaxon:5.6")
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ class InnovationsHauptplatzCodingWeeksCollector : TwoStepEventCollector<Element>
override fun getAllUnparsedEvents(): List<Element> {
val document = Jsoup.parse(fetcher.fetchUrl("https://innovation.linz.at/de/aktuelle-projekte/coding-weeks/"))
return document.select("div#listView div.event")
.asSequence()
.map { it.parent()!! }
.filter { it.attr("href").isNotBlank() }
.map { it.outerHtml() }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import net.fortuna.ical4j.data.CalendarBuilder
import net.fortuna.ical4j.model.Calendar
import net.fortuna.ical4j.model.component.VEvent
import org.jsoup.Jsoup
import java.net.URI
import java.net.URL
import java.time.LocalDate
import java.time.LocalDateTime
Expand All @@ -27,8 +28,8 @@ class JkuEventCollector : EventCollector {

val icsUrls = eventUrls
.flatMap {
val document = Jsoup.parse(fetcher.fetchUrl("https://www.jku.at$it"))
document.select("a").eachAttr("href")
val eventJson = Jsoup.parse(fetcher.fetchUrl("https://www.jku.at$it"))
eventJson.select("a").eachAttr("href")
}
.filter { it.endsWith(".ics") }

Expand All @@ -37,41 +38,40 @@ class JkuEventCollector : EventCollector {

icsUrls.forEach {
val fullUrl = "https://www.jku.at${it}"
val url = URL(fullUrl)
val url = URI(fullUrl).toURL()
events.addAll(parseEventFromIcs(url))
}

return events
}

fun parseEventFromIcs(icsUrl: URL): List<Event> {
private fun parseEventFromIcs(icsUrl: URL): List<Event> {
val formatter = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'")
val daylongEventFormatter = DateTimeFormatter.ofPattern("yyyyMMdd")
icsUrl.openStream().use {
icsUrl.openStream().use { inputStream ->
val builder = CalendarBuilder()
val calendar: Calendar = builder.build(it)
val calendar: Calendar = builder.build(inputStream)
val components = calendar.components.filterIsInstance<VEvent>()

return components.map {
val title = it.summary.value
val eventName = title
val eventStartDate = if (it.isDaylongEvent()) {
val dateTime = LocalDate.parse(it.startDate.value, daylongEventFormatter)
return components.map { vEvent ->
val eventName = vEvent.summary.value
val eventStartDate = if (vEvent.isDaylongEvent()) {
val dateTime = LocalDate.parse(vEvent.startDate.value, daylongEventFormatter)
val zonedDateTime = ZonedDateTime.of(dateTime.atTime(0, 0), ZoneId.of("UTC"))
zonedDateTime.toOffsetDateTime()
} else {
val dateTime = LocalDateTime.parse(it.startDate.value, formatter)
val dateTime = LocalDateTime.parse(vEvent.startDate.value, formatter)
val zonedDateTime = ZonedDateTime.of(dateTime, ZoneId.of("UTC"))
zonedDateTime.toOffsetDateTime()
}

Event(
eventName, eventStartDate,
mapOf(
SemanticKeys.LOCATION_NAME to it.location.value,
SemanticKeys.LOCATION_NAME to vEvent.location.value,
SemanticKeys.TAGS to listOf("JKU", "Universität", "Studieren").toString(),
"url.ics" to icsUrl.toString(),
"jku.uid" to it.uid.value
"jku.uid" to vEvent.uid.value
)
)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ class KupfTicketCollector : TwoStepEventCollector<JsonObject>("kupfticket") {
override fun getAllUnparsedEvents(): List<JsonObject> {
val fetcher = Fetcher()
val document = Jsoup.parse(fetcher.fetchUrl("https://kupfticket.com/events"))
val next_data = document.select("body script#__NEXT_DATA__").first()!!.data()
val jsonObject = Parser.default().parse(StringReader(next_data)) as JsonObject
val nextData = document.select("body script#__NEXT_DATA__").first()!!.data()
val jsonObject = Parser.default().parse(StringReader(nextData)) as JsonObject

val eventSlugs = jsonObject.lookup<String>("props.pageProps.events.edges.node.slug")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,24 +23,24 @@ class LandestheaterLinzCollector :

val document = fetchList(fetcher)
document.select("section")
.forEach {
.forEach { section ->
//sections
val date = parseDateFromSection(it)
it.select("div.lth-section-content > div.lth-evitem")
.forEach {
val link = it.select("div.lth-evitem-title > a")
val date = parseDateFromSection(section)
section.select("div.lth-section-content > div.lth-evitem")
.forEach { evitem ->
val link = evitem.select("div.lth-evitem-title > a")
val linkSeason = link.attr("data-lth-season")
val linkEventSetId = link.attr("data-lth-eventsetid")
val linkRef = link.attr("data-lth-ref")
val url =
"https://www.landestheater-linz.at/stuecke/detail?EventSetID=${linkEventSetId}&ref=${linkRef}&spielzeit=${linkSeason}"
events.add(Triple(it, url, date))
events.add(Triple(evitem, url, date))
}
}

val eventUrls = events
.map {
val link = it.first.select("div.lth-evitem-title > a")
.map { event ->
val link = event.first.select("div.lth-evitem-title > a")
val linkSeason = link.attr("data-lth-season")
val linkEventSetId = link.attr("data-lth-eventsetid")
val linkRef = link.attr("data-lth-ref")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ class LinzTermineCollector : EventCollector {
private fun getEventWebsites(events: List<Event>): Map<String, Document> {
//TODO we may loose some events because either they do not have a linztermine.at link or the linztermine.at link points to a 404... not sure what to do about that
return events
.asSequence()
.map { it.url }
.filter {
it.contains("linztermine.at")
Expand Down Expand Up @@ -65,12 +66,9 @@ class LinzTermineCollector : EventCollector {
LOG.warn("event does not contain any dates: $event")
continue
}
val website = eventWebsites[event.url] ?: continue

val location = locations[event.locationId]
val website = eventWebsites[event.url]
if (website == null) {
continue
}
val description = website.select("span.content-description").text()
val pictureUrl = if (!website.select("div.letterbox > img").isEmpty()) {
"https://www.linztermine.at" + website.select("div.letterbox > img").attr("src")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,11 @@ class OOESeniorenbundCollector : TwoStepEventCollector<Pair<Document, String>>("

val sessionIdPattern = Pattern.compile("\\;jsessionid\\=[\\d\\w]+\\?")
val matcher = sessionIdPattern.matcher(url)
if (matcher.find()) {
return url.replace(matcher.group(0), "?")

return if (matcher.find()) {
url.replace(matcher.group(0), "?")
} else {
return url
url
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,20 +88,25 @@ class PlanetTTCollector : TwoStepEventCollector<Element>("planettt") {

private fun mapLocation(data: MutableMap<String, String>, event: Element) {
val location = event.select("div.pl-modal-location").attr("data-location")
if (location == "simmcity") {
data[SemanticKeys.LOCATION_NAME] = "SiMMCity"
data[SemanticKeys.LOCATION_URL] = "https://simmcity.at/"
data[SemanticKeys.LOCATION_CITY] = "Wien"
} else if (location == "szene") {
data[SemanticKeys.LOCATION_NAME] = "Szene"
data[SemanticKeys.LOCATION_URL] = "https://szene.wien/"
data[SemanticKeys.LOCATION_CITY] = "Wien"
} else if (location == "planet") {
data[SemanticKeys.LOCATION_NAME] = "Gasometer"
data[SemanticKeys.LOCATION_URL] = "https://www.gasometer.at/"
data[SemanticKeys.LOCATION_CITY] = "Wien"
} else {
LOG.warn("could not guess location from location: $location")
when (location) {
"simmcity" -> {
data[SemanticKeys.LOCATION_NAME] = "SiMMCity"
data[SemanticKeys.LOCATION_URL] = "https://simmcity.at/"
data[SemanticKeys.LOCATION_CITY] = "Wien"
}
"szene" -> {
data[SemanticKeys.LOCATION_NAME] = "Szene"
data[SemanticKeys.LOCATION_URL] = "https://szene.wien/"
data[SemanticKeys.LOCATION_CITY] = "Wien"
}
"planet" -> {
data[SemanticKeys.LOCATION_NAME] = "Gasometer"
data[SemanticKeys.LOCATION_URL] = "https://www.gasometer.at/"
data[SemanticKeys.LOCATION_CITY] = "Wien"
}
else -> {
LOG.warn("could not guess location from location: $location")
}
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ class StadtwerkstattCollector : TwoStepEventCollector<String>("stadtwerkstatt")

var name = eventSite.select("li.event-title").text()
if (name.isBlank()) {
name = eventSite.select("ul.event-artists span.name").map { it.text().trim() }.joinToString(", ")
name = eventSite.select("ul.event-artists span.name").joinToString(", ") { it.text().trim() }
}
val startDate = parseDate(eventSite)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,25 +1,27 @@
package events.boudicca.eventcollector.collectors

import com.rometools.rome.feed.synd.SyndFeed
import com.rometools.rome.io.SyndFeedInput
import com.rometools.rome.io.XmlReader
import events.boudicca.SemanticKeys
import events.boudicca.api.eventcollector.Event
import events.boudicca.api.eventcollector.EventCollector
import java.net.URL
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClients
import java.time.LocalDateTime
import java.time.ZoneId
import java.time.ZonedDateTime
import java.time.format.DateTimeFormatter


class TechnologiePlauscherlCollector : EventCollector {
override fun getName(): String {
return "technologieplauscherl"
}

override fun collectEvents(): List<Event> {
val url = URL("https://technologieplauscherl.at/feed")
val input = SyndFeedInput()
val feed = input.build(XmlReader(url))
val url = "https://technologieplauscherl.at/feed"
val feed = prepareSyndFeed(url)

val events = feed.entries.map { entry ->

Expand Down Expand Up @@ -49,4 +51,14 @@ class TechnologiePlauscherlCollector : EventCollector {

return events
}

private fun prepareSyndFeed(url: String): SyndFeed {
HttpClients.createMinimal().use { client ->
client.execute(HttpGet(url)).use { response ->
response.entity.content.use { stream ->
return SyndFeedInput().build(XmlReader(stream))
}
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ class UlfOoeCollector : TwoStepEventCollector<String>("ulfooe") {
val regex = """(?<name>.*?)[,|\s]*(?<zip>\d{4}) (?<city>[\w\s]+)""".toRegex()
val matchResult = regex.find(locationName)
if (matchResult != null) {
data[SemanticKeys.LOCATION_NAME] = matchResult.groups.get("name")!!.value.trimEnd()
data[SemanticKeys.LOCATION_CITY] = matchResult.groups.get("city")!!.value.trimEnd()
data[SemanticKeys.LOCATION_NAME] = matchResult.groups["name"]!!.value.trimEnd()
data[SemanticKeys.LOCATION_CITY] = matchResult.groups["city"]!!.value.trimEnd()
} else {
data[SemanticKeys.LOCATION_NAME] = locationName
}
Expand Down
16 changes: 8 additions & 8 deletions eventdb-openapi/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,20 @@ val openapi by configurations.creating {
isCanBeResolved = true
}

val jackson_version = "2.15.2"
val jakarta_annotation_version = "1.3.5"
val jacksonVersion = "2.15.2"
val jakartaAnnotationVersion = "1.3.5"

dependencies {
// openapi(project(mapOf("configuration" to "openapi", "path" to ":eventdb")))
openapi(files("src/main/resources/openapi.yml"))

implementation("com.google.code.findbugs:jsr305:3.0.2")
implementation("com.fasterxml.jackson.core:jackson-core:$jackson_version")
implementation("com.fasterxml.jackson.core:jackson-annotations:$jackson_version")
implementation("com.fasterxml.jackson.core:jackson-databind:$jackson_version")
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$jackson_version")
implementation("com.fasterxml.jackson.core:jackson-core:$jacksonVersion")
implementation("com.fasterxml.jackson.core:jackson-annotations:$jacksonVersion")
implementation("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion")
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$jacksonVersion")
implementation("org.openapitools:jackson-databind-nullable:0.2.6")
implementation("jakarta.annotation:jakarta.annotation-api:$jakarta_annotation_version")
implementation("jakarta.annotation:jakarta.annotation-api:$jakartaAnnotationVersion")
api(project(":semantic-conventions"))
}

Expand All @@ -48,7 +48,7 @@ tasks.withType<org.openapitools.generator.gradle.plugin.tasks.GenerateTask> {
sourceSets {
main {
java {
srcDir(file("$buildDir/generate-resources/main/src/main/java"))
srcDir(file("${layout.buildDirectory}/generate-resources/main/src/main/java"))
}
}
}
Expand Down
Loading

0 comments on commit f2eaead

Please sign in to comment.