Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrate build scripts to use ymlstash #66

Merged
merged 8 commits into from
May 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.10"]
python-version: ["3.12"]

steps:
- uses: actions/checkout@v3
Expand Down
1 change: 1 addition & 0 deletions Pipfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ requests = "*"
aiohttp = "*"
unidecode = "*"
click = "*"
ymlstash = "*"

[dev-packages]
black = "*"
Expand Down
1,188 changes: 623 additions & 565 deletions Pipfile.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

Open a new pull request to this repository that adds a new YAML file to the [`names`](names) directory.

If your domain name is `examp.le` the file name should be `example.yml`, and the following minimal fields are required:
If your domain name is `examp.le` the file name should be `examp.le.yml`, and the following minimal fields are required:

```yaml
domain: examp.le
Expand Down
15 changes: 12 additions & 3 deletions cli/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
import os

from .base import cli

from .build import build
from .clean import clean
from .ghpr import ghpr
from .scan import scan

__all__ = [cli, build]

# ugly workaround to avoid build failures in Python 3.8
if "NETLIFY" not in os.environ:
from .clean import clean
from .ghpr import ghpr
from .scan import scan

__all__ += [clean, ghpr, scan]
9 changes: 9 additions & 0 deletions cli/base.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,13 @@
import click
import ymlstash

from pathlib import Path
from .model import Name

ROOT_PATH = Path(__file__).parents[1]
NAMES_DIR = ROOT_PATH / "names"

STASH = ymlstash.YmlStash(Name, NAMES_DIR, filter_none=True)


@click.group
Expand Down
47 changes: 15 additions & 32 deletions cli/build.py
Original file line number Diff line number Diff line change
@@ -1,22 +1,17 @@
import yaml

from jinja2 import Environment, PackageLoader, select_autoescape
from os import listdir, mkdir
from pathlib import Path
from shutil import copyfile
from unidecode import unidecode

from .base import cli
from .base import cli, STASH

ROOT_PATH = Path(__file__).parents[1]

NAMES_DIR = ROOT_PATH / "names"
BUILD_DIR = ROOT_PATH / "build"
STATIC_DIR = ROOT_PATH / "static"

REQUIRED_FIELDS = set(["domain", "name"])
OPTIONAL_FIELDS = set(["url", "title", "email", "github", "candidate", "invalid"])

TEMPLATES = ["index.html"]


Expand All @@ -32,38 +27,26 @@ def build():
names = []
candidates = []

for name in listdir(NAMES_DIR):
with open(NAMES_DIR / name, "r") as f:
fields = yaml.load(f.read(), Loader=yaml.Loader)
field_set = set(fields.keys())

missing_fields = REQUIRED_FIELDS - field_set
if missing_fields:
raise Exception(f"Missing required fields {missing_fields} in {name}")

invalid_fields = field_set - OPTIONAL_FIELDS - REQUIRED_FIELDS
if invalid_fields:
raise Exception(f"Invalid fields {invalid_fields} in {name}")

if fields.get("invalid"):
continue

if fields.get("candidate"):
candidates.append(fields)
else:
names.append(fields)
for key in STASH.list_keys():
name = STASH.load(key)
if name.invalid is True:
continue
if name.candidate is True:
candidates.append(name)
else:
names.append(name)

names = list(sorted(names, key=lambda x: x["domain"]))
candidates = list(sorted(candidates, key=lambda x: x["domain"]))
names = list(sorted(names, key=lambda x: x.domain))
candidates = list(sorted(candidates, key=lambda x: x.domain))

def render_link(value, classes):
name = unidecode(value["name"]).lower().split(" ")
domain = unidecode(value["domain"]).replace(".", "")
name = unidecode(value.name).lower().split(" ")
domain = unidecode(value.domain).replace(".", "")
res = []
for part in name:
if part == domain:
url = value.get("url") or "https://" + value.get("domain")
res.append(f'<a href="{url}" class="{classes}">{value["domain"]}</a>')
url = value.url or "https://" + value.domain
res.append(f'<a href="{url}" class="{classes}">{value.domain}</a>')
else:
res.append(part)
return " ".join(res)
Expand Down
7 changes: 5 additions & 2 deletions cli/clean.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
from .base import cli
from .base import cli, STASH


@cli.command()
def clean():
print("Cleaning..")
names = STASH.list_keys()
for name in names:
obj = STASH.load(name)
STASH.save(obj)
5 changes: 1 addition & 4 deletions cli/ghpr.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import click
import sys

from pathlib import Path
from subprocess import run

from .base import cli

Expand All @@ -14,8 +12,7 @@
def ghpr(path):
with open(path, "r") as f:
for line in f:
l = line.strip().split(",")
domain, handle, name = [x.strip() for x in l]
domain, handle, name = [x.strip() for x in line.strip().split(",")]
handle = None if "@" in handle else handle

fn = domain.replace(".", "")
Expand Down
15 changes: 15 additions & 0 deletions cli/model.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
from dataclasses import dataclass
from typing import ClassVar, Optional


@dataclass
class Name:
domain: str
name: str
title: Optional[str] = None
url: Optional[str] = None
email: Optional[str] = None
github: Optional[str] = None
candidate: Optional[bool] = None
invalid: Optional[bool] = None
key: ClassVar[str] = "domain"
6 changes: 3 additions & 3 deletions cli/scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def fetch_tlds(self):
tld.lower() for tld in res.text.strip().split("\n")[1:] if len(tld) < 4
]
f.write("\n".join(tlds).strip())
TLDS = set(tlds)
_TLDS = set(tlds)

def fetch_homepage(self, domain):
print(f"Fetching {domain}...", end="")
Expand Down Expand Up @@ -92,7 +92,7 @@ async def get_homepage(self, session, domain):
with open(DATA_DIR / "homepages" / f"{domain}.html", "w") as out:
out.write(res)
return len(res)
except Exception as e:
except Exception:
return 0

async def fetch_homepages(self, N):
Expand All @@ -105,7 +105,7 @@ async def fetch_homepages(self, N):
for name in names:
tasks.append(asyncio.ensure_future(self.get_homepage(session, name)))

resps = await asyncio.gather(*tasks)
_resps = await asyncio.gather(*tasks)

def find_homepages(self):
res = {}
Expand Down
1 change: 0 additions & 1 deletion manage.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import os
import click

from cli import cli
Expand Down
2 changes: 1 addition & 1 deletion names/abraham.yml → names/abrah.am.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
domain: abrah.am
name: Abraham Williams
title: Developer, conference speaker, and open source maintainer.
github: abraham
github: abraham
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
4 changes: 0 additions & 4 deletions names/cesar.yml

This file was deleted.

File renamed without changes.
2 changes: 1 addition & 1 deletion names/ciechanowski.yml → names/ciechanow.ski.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
domain: ciechanow.ski
name: Bartosz Ciechanowski
title: Writes interactive articles
title: Writes interactive articles
File renamed without changes.
File renamed without changes.
2 changes: 1 addition & 1 deletion names/conrad.yml → names/conr.ad.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
domain: conr.ad
name: Conrad Decker
github: conraddecker
title: Digital Creator & Technologist
url: https://conr.ad
github: conraddecker
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
3 changes: 3 additions & 0 deletions names/feli.pe.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
domain: feli.pe
name: Felipe Oduardo Sierra
title: Father. Software Engineer. Hopeful Traveler.
3 changes: 0 additions & 3 deletions names/felipe.yml

This file was deleted.

File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
2 changes: 1 addition & 1 deletion names/itai.yml → names/it.ai.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
domain: it.ai
name: Itai Nathaniel
email: [email protected]
title: Developer, pâtissier, father
email: [email protected]
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
4 changes: 4 additions & 0 deletions names/josh.ua.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
domain: josh.ua
name: Joshua Bell
title: 'Software Engineer (full-stack): mobile, web, blockchain.'
github: joshuabell
4 changes: 0 additions & 4 deletions names/joshua.yml

This file was deleted.

File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
2 changes: 1 addition & 1 deletion names/nihar.yml → names/nih.ar.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
domain: nih.ar
name: Nihar Samantaray
email: [email protected]
title: Yet another hobbyist systems programmer
email: [email protected]
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Loading
Loading