Skip to content

Commit

Permalink
added minor changes, cleaned up code
Browse files Browse the repository at this point in the history
  • Loading branch information
alexandrebrilhante committed Jun 19, 2024
1 parent 600b2b4 commit 3775d5e
Show file tree
Hide file tree
Showing 5 changed files with 18 additions and 64 deletions.
40 changes: 10 additions & 30 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ This software comes bundled with a Grafana dashboard to observe the message queu

## Dependencies

Ensure your `Cargo.toml` file includes dependencies for the `tokio` and `pulsar-rs` crates:
Ensure your `Cargo.toml` file includes dependencies for the `pulsar-rs` and `tokio` crates:

```toml
[dependencies]
Expand All @@ -30,46 +30,35 @@ cargo build --release && cargo run --release

#### Cassandra

Ensure your Cassandra cluster is setup:
Ensure your Cassandra cluster is setup then start Apache Pulsar and the `tick_handler` executable:

```bash
CREATE KEYSPACE pulsar_test_keyspace WITH replication = {'class':'SimpleStrategy', 'replication_factor':1};
CREATE KEYSPACE pulsar_keyspace WITH replication = {'class':'SimpleStrategy', 'replication_factor':1};

USE pulsar_test_keyspace;
USE pulsar_keyspace;

CREATE TABLE pulsar_test_table (key text PRIMARY KEY, col text);
CREATE TABLE pulsar_cassandra_sink (key text PRIMARY KEY, col text);
````

```bash
pulsar-daemon start standalone
docker run -d --rm --name=cassandra -p 9042:9042 cassandra:latest
docker exec -ti cassandra cqlsh localhost
CREATE KEYSPACE pulsar_test_keyspace WITH replication = {'class':'SimpleStrategy', 'replication_factor':1};
USE pulsar_test_keyspace;
CREATE TABLE pulsar_test_table (key text PRIMARY KEY, col text);
pulsar-admin sinks create \
--tenant public \
--namespace default \
--name cassandra-test-sink \
--name cassandra-sink \
--archive $PWD/pulsar/connectors/pulsar-io-cassandra-3.2.2.nar \
--sink-config-file $PWD/pulsar/connectors/cassandra-sink.yml \
--inputs test
```

#### PostgreSQL

Ensure your PostgreSQL database is setup:
Ensure your PostgreSQL database is setup then start Apache Pulsar and the `tick_handler` executable:

```sql
psql -U postgres postgres
CREATE TABLE IF NOT EXISTS pulsar_postgres_jdbc_sink (
id serial PRIMARY KEY,
name TEXT NOT NULL
Expand All @@ -79,21 +68,12 @@ CREATE TABLE IF NOT EXISTS pulsar_postgres_jdbc_sink (
```bash
pulsar-daemon start standalone
docker pull postgres:12
docker run -d -it --rm \
--name pulsar-postgres \
-p 5432:5432 \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_USER=postgres \
postgres:latest
pulsar-admin schemas upload pulsar-postgres-jdbc-sink-topic -f ./connectors/avro-schema
pulsar-admin schemas upload postgres-sink-topic -f ./connectors/avro-schema
pulsar-admin sinks create \
--archive $PWD/pulsar/connectors/pulsar-io-jdbc-postgres-3.2.2.nar \
--inputs test \
--name pulsar-postgres-jdbc-sink \
--sink-config-file $PWD/pulsar/connectors/pulsar-postgres-jdbc-sink.yaml \
--name postgres-sink \
--sink-config-file $PWD/pulsar/connectors/postgres-sink.yaml \
--parallelism 1
```
4 changes: 2 additions & 2 deletions pulsar/connectors/cassandra-sink.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
configs:
roots: "localhost:9042"
keyspace: "pulsar_test_keyspace"
columnFamily: "pulsar_test_table"
keyspace: "pulsar_keyspace"
columnFamily: "pulsar_cassandra_sink"
keyname: "key"
columnName: "col"
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
configs:
userName: "alexandre-brilhante"
userName: "postgres"
password: "postgres"
jdbcUrl: "jdbc:postgresql://localhost:5432/postgres"
tableName: "pulsar_postgres_jdbc_sink"
10 changes: 1 addition & 9 deletions scripts/run_cassandra.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,10 @@ pulsar-daemon start standalone

docker run -d --rm --name=cassandra -p 9042:9042 cassandra:latest

docker exec -ti cassandra cqlsh localhost

CREATE KEYSPACE pulsar_test_keyspace WITH replication = {'class':'SimpleStrategy', 'replication_factor':1};

USE pulsar_test_keyspace;

CREATE TABLE pulsar_test_table (key text PRIMARY KEY, col text);

pulsar-admin sinks create \
--tenant public \
--namespace default \
--name cassandra-test-sink \
--name cassandra-sink \
--archive $PWD/pulsar/connectors/pulsar-io-cassandra-3.2.2.nar \
--sink-config-file $PWD/pulsar/connectors/cassandra-sink.yml \
--inputs test
26 changes: 4 additions & 22 deletions scripts/run_postgres.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,30 +2,12 @@

pulsar-daemon start standalone

docker pull postgres:12

docker run -d -it --rm \
--name pulsar-postgres \
-p 5432:5432 \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_USER=postgres \
postgres:latest

docker exec -it pulsar-postgres /bin/bash

psql -U postgres postgres

create table if not exists pulsar_postgres_jdbc_sink
(
id serial PRIMARY KEY,
name VARCHAR(255) NOT NULL
);

pulsar-admin schemas upload pulsar-postgres-jdbc-sink-topic -f ./connectors/avro-schema
pulsar-admin schemas upload postgres-sink-topic -f ./connectors/avro-schema

pulsar-admin sinks create \
--archive $PWD/pulsar/connectors/pulsar-io-jdbc-postgres-3.2.2.nar \
--inputs test \
--name pulsar-postgres-jdbc-sink \
--sink-config-file $PWD/pulsar/connectors/pulsar-postgres-jdbc-sink.yaml \
--name postgres-sink \
--sink-config-file $PWD/pulsar/connectors/postgres-sink.yaml \
--parallelism 1

0 comments on commit 3775d5e

Please sign in to comment.