Skip to content
This repository has been archived by the owner on Jan 8, 2024. It is now read-only.

Commit

Permalink
Atom api with java 19 (#23)
Browse files Browse the repository at this point in the history 
* Java 19 with sync api

Signed-off-by: Prabhu Subramanian <[email protected]>

---------

Signed-off-by: Prabhu Subramanian <[email protected]>
  • Loading branch information
@prabhu
prabhu authored Jun 6, 2023
1 parent e2dea09 commit 68900b7
Show file tree
Hide file tree
Showing 9 changed files with 2,093 additions and 2,768 deletions.
2 changes: 2 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,8 @@ Adding asyncio.sleep(0) seems to fix such errors.
await asyncio.sleep(0)
```

Alternatively, use the sync api.

### pygraphviz refuses to install

```
Expand Down
12 changes: 8 additions & 4 deletions api_docs/joern_lib/client.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ Functions
---------


`bulk_query(connection, query_list)`
`bulk_query(connection, query_list, sync=False)`
: Bulk query joern server


Expand All @@ -33,19 +33,23 @@ Functions
: Function to create a connection to joern and cpggen server


`get_sync(base_url='http://localhost:9000', cpggen_url='http://localhost:7072', username=None, password=None)`
: Function to create a plain synchronous http connection to joern and cpggen server


`graphml_export(connection, filter_str='method')`
: Method to export method as graphml


`p(connection, query_str, title='', caption='')`
`p(connection, query_str, title='', caption='', sync=False)`
: Function to print the result as a table


`q(connection, query_str)`
`q(connection, query_str, sync=False)`
: Query joern server and optionally print the result as a table if the query ends with .p


`query(connection, query_str)`
`query(connection, query_str, sync=False)`
: Query joern server


Expand Down
6 changes: 5 additions & 1 deletion api_docs/joern_lib/workspace.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,11 @@ Functions
: Function to check if a CPG exists in the workspace


`create_cpg(connection, src, out_dir=None, languages='autodetect', project_name=None, slice=None, slice_mode='Usages', auto_build=True, skip_sbom=True)`
`create_atom(connection, src, out_dir=None, languages='autodetect', project_name=None)`
: Function to create atom using atomgen server


`create_cpg(connection, src, out_dir=None, languages='autodetect', project_name=None, slice=None, slice_mode='Usages', auto_build=True, skip_sbom=True, use_atom=False)`
: Function to create CPG using cpggen server


Expand Down
26 changes: 18 additions & 8 deletions contrib/polynote/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,32 @@ ARG SCALA_VERSION="2.12"
ARG DIST_TAR="polynote-dist.tar.gz"
ARG TARGETPLATFORM

ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk \
PATH=${PATH}:/root/.local/bin:
ENV ATOM_VERSION=1.0.0 \
ATOM_HOME=/opt/atom-1.0.0 \
JAVA_HOME="/opt/graalvm-ce-java19-22.3.1" \
POLYNOTE_SCALA_VERSION=${SCALA_VERSION} \
POLYNOTE_VERSION=${POLYNOTE_VERSION} \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PYTHONIOENCODING="utf-8" \
PATH=${PATH}:/root/.local/bin:/opt/atom-1.0.0/bin:/opt/graalvm-ce-java19-22.3.1/bin:
WORKDIR /opt

RUN set -e; \
ARCH_NAME="$(rpm --eval '%{_arch}')"; \
microdnf update -y && microdnf install -y gcc git-core python3.11 python3.11-devel python3.11-pip which wget tar zip \
unzip sudo graphviz graphviz-gd graphviz-python3 java-17-openjdk-devel \
unzip sudo graphviz graphviz-gd graphviz-python3 \
&& alternatives --install /usr/bin/python3 python /usr/bin/python3.11 1 \
&& python3 --version \
&& python3 -m pip install --upgrade pip \
&& bash <(curl -sL https://get.graalvm.org/jdk) --to /opt graalvm-ce-java19-22.3.1 \
&& java -version \
&& curl -LO https://github.com/AppThreat/atom/releases/latest/download/atom.zip \
&& curl -LO https://github.com/AppThreat/atom/releases/latest/download/atom.zip.sha512 \
&& echo "$(cat atom.zip.sha512 | cut -d ' ' -f1) atom.zip" | sha512sum -c \
&& unzip -q atom.zip -d /opt/ \
&& rm atom.zip \
&& ln -s /opt/atom-${ATOM_VERSION}/bin/atom /usr/local/bin/atom \
&& curl -LO https://repo.almalinux.org/almalinux/9/CRB/${ARCH_NAME}/os/Packages/graphviz-devel-2.44.0-25.el9.${ARCH_NAME}.rpm \
&& rpm -ivh graphviz-devel-2.44.0-25.el9.${ARCH_NAME}.rpm \
&& rm graphviz-devel-2.44.0-25.el9.${ARCH_NAME}.rpm \
Expand All @@ -44,10 +59,5 @@ RUN set -e; \
&& microdnf clean all

EXPOSE 8192
ENV POLYNOTE_SCALA_VERSION=${SCALA_VERSION} \
POLYNOTE_VERSION=${POLYNOTE_VERSION} \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PYTHONIOENCODING="utf-8"

ENTRYPOINT ["/opt/polynote/polynote.py"]
2,036 changes: 1,011 additions & 1,025 deletions contrib/polynote/notebooks/joern-notes/maven.ipynb
View file

Large diffs are not rendered by default.

2,651 changes: 947 additions & 1,704 deletions contrib/polynote/notebooks/joern-notes/vuln-spring.ipynb
View file

Large diffs are not rendered by default.

101 changes: 77 additions & 24 deletions joern_lib/client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,24 @@ async def __aexit__(self, exc_type, exc_value, exc_traceback):
return await self.close()


def get_sync(
base_url="http://localhost:9000",
cpggen_url="http://localhost:7072",
username=None,
password=None,
):
"""Function to create a plain synchronous http connection to joern and cpggen server"""
auth = None
if username and password:
auth = httpx.BasicAuth(username, password)
base_url = base_url.rstrip("/")
client = httpx.Client(base_url=base_url, auth=auth, timeout=CLIENT_TIMEOUT)
cpggenclient = None
if cpggen_url:
cpggenclient = httpx.Client(base_url=cpggen_url, timeout=CLIENT_TIMEOUT)
return Connection(cpggenclient, client, None)


async def get(
base_url="http://localhost:9000",
cpggen_url="http://localhost:7072",
Expand All @@ -76,7 +94,7 @@ async def get(
if cpggen_url:
cpggenclient = httpx.AsyncClient(base_url=cpggen_url, timeout=CLIENT_TIMEOUT)
ws_url = f"""{base_url.replace("http://", "ws://").replace("https://", "wss://")}/connect"""
websocket = await websockets.connect(ws_url, ping_timeout=None)
websocket = await websockets.connect(ws_url, ping_interval=None, ping_timeout=None)
connected_msg = await websocket.recv()
if connected_msg != "connected":
raise websockets.exceptions.InvalidState(
Expand All @@ -97,31 +115,41 @@ async def receive(connection):
return await connection.websocket.recv()


async def p(connection, query_str, title="", caption=""):
async def p(connection, query_str, title="", caption="", sync=False):
"""Function to print the result as a table"""
result = await query(connection, query_str)
result = await query(connection, query_str, sync=sync)
print_table(result, title, caption)
return result


async def q(connection, query_str):
async def q(connection, query_str, sync=False):
"""Query joern server and optionally print the result as a table if the query ends with .p"""
if query_str.strip().endswith(".p"):
query_str = f"{query_str[:-2]}.toJsonPretty"
return await p(connection, query_str)
return await query(connection, query_str)
return await p(connection, query_str, sync=sync)
return await query(connection, query_str, sync=sync)


async def query(connection, query_str):
async def query(connection, query_str, sync=False):
"""Query joern server"""
client = connection.httpclient
response = await client.post(
url="/query",
headers=headers,
json={"query": fix_query(query_str)},
)
if isinstance(client, httpx.AsyncClient):
response = await client.post(
url=f"/query{'-sync' if sync else ''}",
headers=headers,
json={"query": fix_query(query_str)},
)
else:
sync = True
response = client.post(
url=f"/query{'-sync' if sync else ''}",
headers=headers,
json={"query": fix_query(query_str)},
)
if response.status_code == httpx.codes.OK:
j = response.json()
if sync:
return fix_json(j.get("stdout", ""))
res_uuid = j.get("uuid")
try:
completed_uuid = await receive(connection)
Expand All @@ -141,20 +169,38 @@ async def query(connection, query_str):
return None


async def bulk_query(connection, query_list):
async def bulk_query(connection, query_list, sync=False):
"""Bulk query joern server"""
client = connection.httpclient
websocket = connection.websocket
uuid_list = []
response_list = []
for query_str in query_list:
response = await client.post(
url="/query", headers=headers, json={"query": fix_query(query_str)}
)
if isinstance(client, httpx.AsyncClient):
response = await client.post(
url=f"/query{'-sync' if sync else ''}",
headers=headers,
json={"query": fix_query(query_str)},
)
else:
sync = True
response = client.post(
url="/query-sync", headers=headers, json={"query": fix_query(query_str)}
)
if response.status_code == httpx.codes.OK:
j = response.json()
res_uuid = j.get("uuid")
uuid_list.append(res_uuid)
if sync:
sout = j.get("stdout", "")
serr = j.get("stderr", "")
if sout:
response_list.append(fix_json(sout))
else:
response_list.append({"error": parse_error(serr)})
else:
res_uuid = j.get("uuid")
uuid_list.append(res_uuid)
if sync:
return response_list
async for completed_uuid in websocket:
if completed_uuid in uuid_list:
response = await client.get(
Expand All @@ -179,7 +225,7 @@ async def flows(connection, source, sink):
connection,
source,
sink,
print_result=True if os.getenv("POLYNOTE_VERSION") else False,
print_result=bool(os.getenv("POLYNOTE_VERSION")),
)


Expand Down Expand Up @@ -314,11 +360,18 @@ async def create_cpg(
"auto_build": "true" if auto_build else "false",
"skip_sbom": "true" if skip_sbom else "false",
}
response = await client.post(
url="/cpg",
headers=headers,
json=data,
)
if isinstance(client, httpx.AsyncClient):
response = await client.post(
url="/cpg",
headers=headers,
json=data,
)
else:
response = client.post(
url="/cpg",
headers=headers,
json=data,
)
return response.json()


Expand Down
25 changes: 24 additions & 1 deletion joern_lib/workspace.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,33 @@ async def slice_cpg(
try:
with open(app_manifests.get("slice_out"), encoding="utf-8") as fp:
return json.load(fp)
except Exception:
except json.JSONDecodeError:
return None
return None


async def create_atom(
connection,
src,
out_dir=None,
languages="autodetect",
project_name=None,
):
"""Function to create atom using atomgen server"""
return await create_cpg(
connection,
src,
out_dir=out_dir,
languages=languages,
project_name=project_name,
slice=False,
slice_mode="Usages",
auto_build=True,
skip_sbom=False,
use_atom=True,
)


async def create_cpg(
connection,
src,
Expand All @@ -66,6 +88,7 @@ async def create_cpg(
slice_mode="Usages",
auto_build=True,
skip_sbom=True,
use_atom=False,
):
"""Function to create CPG using cpggen server"""
app_manifests = []
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "joern-lib"
version = "0.11.1"
version = "0.12.0"
description = "Python library for code analysis with CPG and Joern"
authors = ["Team AppThreat <[email protected]>"]
license = "Apache-2.0"
Expand Down

0 comments on commit 68900b7

Please sign in to comment.