diff --git a/MANIFEST.in b/MANIFEST.in index 96e482f1..f58e7d8c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ recursive-include bert_e/templates * +recursive-include bert_e/server/templates * +recursive-include bert_e/server/static * recursive-include bert_e/docs * include requirements.txt diff --git a/README.md b/README.md index 8b8cdcc4..bb49c5c7 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ All above instructions will assume you are inside the codespace environment ```shell $ cp settings.sample.yml settings.yml # Configure settings.yml to your liking -$ tox -e run +$ tox run -e run ``` ### Run local tests diff --git a/bert_e/git_host/github/schema.py b/bert_e/git_host/github/schema.py index d0c46b2b..fda64620 100644 --- a/bert_e/git_host/github/schema.py +++ b/bert_e/git_host/github/schema.py @@ -18,10 +18,15 @@ used by Bert-E) are declared. """ -from marshmallow import Schema, fields +from marshmallow import Schema, fields, EXCLUDE -class User(Schema): +class GitHubSchema(Schema): + class Meta: + unknown = EXCLUDE + + +class User(GitHubSchema): id = fields.Int(required=True) login = fields.Str(required=True) # Note: the "printable" name can be absent in most API call results. @@ -30,7 +35,7 @@ class User(Schema): type = fields.Str() -class Repo(Schema): +class Repo(GitHubSchema): name = fields.Str(required=True) owner = fields.Nested(User, required=True) full_name = fields.Str(required=True) @@ -41,7 +46,7 @@ class Repo(Schema): default_branch = fields.Str() -class CreateRepo(Schema): +class CreateRepo(GitHubSchema): name = fields.Str(required=True) description = fields.Str() homepage = fields.Url() @@ -54,14 +59,14 @@ class CreateRepo(Schema): licence_template = fields.Str() -class Status(Schema): +class Status(GitHubSchema): state = fields.Str(required=True) target_url = fields.Str(required=True, allow_none=True) description = fields.Str(required=True, allow_none=True) context = fields.Str(required=True) -class AggregatedStatus(Schema): +class AggregatedStatus(GitHubSchema): # The most convenient way to get a pull request's build status is to # query github's API for an aggregated status. state = fields.Str() @@ -70,7 +75,7 @@ class AggregatedStatus(Schema): statuses = fields.Nested(Status, many=True, required=True) -class Branch(Schema): +class Branch(GitHubSchema): label = fields.Str() # user:ref or org:ref ref = fields.Str() sha = fields.Str() @@ -78,7 +83,7 @@ class Branch(Schema): repo = fields.Nested(Repo) -class App(Schema): +class App(GitHubSchema): id = fields.Int() slug = fields.Str() owner = fields.Nested(User) @@ -86,7 +91,7 @@ class App(Schema): description = fields.Str() -class CheckSuite(Schema): +class CheckSuite(GitHubSchema): id = fields.Integer() head_sha = fields.Str() head_branch = fields.Str() @@ -98,12 +103,12 @@ class CheckSuite(Schema): app = fields.Nested(App) -class AggregateCheckSuites(Schema): +class AggregateCheckSuites(GitHubSchema): total_count = fields.Integer() check_suites = fields.Nested(CheckSuite, many=True) -class CheckRun(Schema): +class CheckRun(GitHubSchema): id = fields.Integer() head_sha = fields.Str() status = fields.Str() @@ -111,7 +116,7 @@ class CheckRun(Schema): html_url = fields.Url() -class WorkflowRun(Schema): +class WorkflowRun(GitHubSchema): id = fields.Integer() head_sha = fields.Str() head_branch = fields.Str() @@ -121,17 +126,17 @@ class WorkflowRun(Schema): event = fields.Str() -class AggregateWorkflowRuns(Schema): +class AggregateWorkflowRuns(GitHubSchema): total_count = fields.Integer() workflow_runs = fields.Nested(WorkflowRun, many=True) -class AggregateCheckRuns(Schema): +class AggregateCheckRuns(GitHubSchema): total_count = fields.Integer() check_runs = fields.Nested(CheckRun, many=True) -class PullRequest(Schema): +class PullRequest(GitHubSchema): number = fields.Int(required=True) url = fields.Url() html_url = fields.Url() @@ -149,7 +154,7 @@ class PullRequest(Schema): merged_at = fields.DateTime(allow_none=True) -class CreatePullRequest(Schema): +class CreatePullRequest(GitHubSchema): title = fields.Str(required=True) head = fields.Str(required=True) base = fields.Str(required=True) @@ -157,7 +162,7 @@ class CreatePullRequest(Schema): maintainer_can_modify = fields.Bool() -class UpdatePullRequest(Schema): +class UpdatePullRequest(GitHubSchema): title = fields.Str() body = fields.Str() state = fields.Str() @@ -165,7 +170,7 @@ class UpdatePullRequest(Schema): maintainer_can_modify = fields.Bool() -class Comment(Schema): +class Comment(GitHubSchema): id = fields.Int(required=True) body = fields.Str() created_at = fields.DateTime() @@ -174,11 +179,11 @@ class Comment(Schema): url = fields.Url() -class CreateComment(Schema): +class CreateComment(GitHubSchema): body = fields.Str(required=True) -class Review(Schema): +class Review(GitHubSchema): id = fields.Int(allow_none=True) body = fields.Str(allow_none=True) commit_id = fields.Str() @@ -186,41 +191,41 @@ class Review(Schema): user = fields.Nested(User) -class DraftReview(Schema): +class DraftReview(GitHubSchema): path = fields.Str() position = fields.Int() body = fields.Str() -class CreateReview(Schema): +class CreateReview(GitHubSchema): body = fields.Str(allow_none=True) event = fields.Str() -class PullRequestEvent(Schema): +class PullRequestEvent(GitHubSchema): action = fields.Str(required=True) number = fields.Int() pull_request = fields.Nested(PullRequest) -class Issue(Schema): +class Issue(GitHubSchema): number = fields.Int() title = fields.Str() # If this dict is present and non-empty, then the issue is a pull request. pull_request = fields.Dict(optional=True, default={}) -class IssueCommentEvent(Schema): +class IssueCommentEvent(GitHubSchema): action = fields.Str() issue = fields.Nested(Issue) -class PullRequestReviewEvent(Schema): +class PullRequestReviewEvent(GitHubSchema): action = fields.Str() pull_request = fields.Nested(PullRequest) -class StatusEvent(Schema): +class StatusEvent(GitHubSchema): sha = fields.Str() state = fields.Str() context = fields.Str() @@ -228,7 +233,7 @@ class StatusEvent(Schema): target_url = fields.Str(allow_none=True) -class CheckSuiteEvent(Schema): +class CheckSuiteEvent(GitHubSchema): action = fields.Str() check_suite = fields.Nested(CheckSuite) repository = fields.Nested(Repo) diff --git a/bert_e/lib/schema.py b/bert_e/lib/schema.py index c4f6056a..accf29c9 100644 --- a/bert_e/lib/schema.py +++ b/bert_e/lib/schema.py @@ -37,8 +37,9 @@ def load(cls: Schema, data, **kwargs): the result of any @post_load processing. """ - res, errors = cls(**kwargs).load(data) - if errors: + try: + res = cls(**kwargs).load(data) + except Exception as errors: raise SchemaError(errors) return res diff --git a/bert_e/server/__init__.py b/bert_e/server/__init__.py index ec30ec6b..4b5ec562 100644 --- a/bert_e/server/__init__.py +++ b/bert_e/server/__init__.py @@ -40,8 +40,6 @@ def setup_bert_e(settings_file, debug): """Create and configure Bert-E instance.""" settings = setup_settings(settings_file) - settings['robot_password'] = os.environ['BERT_E_GITHOST_PWD'] - settings['jira_token'] = os.environ['BERT_E_JIRA_TOKEN'] settings['backtrace'] = True bert_e = BertE(settings) diff --git a/bert_e/settings.py b/bert_e/settings.py index 2263e835..3c99cb7f 100644 --- a/bert_e/settings.py +++ b/bert_e/settings.py @@ -4,7 +4,8 @@ import logging import os from marshmallow import ( - Schema, fields, post_load, pre_load, validates_schema, ValidationError) + Schema, fields, post_load, pre_load, validates_schema, ValidationError, + EXCLUDE) from bert_e.exceptions import (IncorrectSettingsFile, SettingsFileNotFound, @@ -122,6 +123,9 @@ def deserialize(self, value, attr=None, data=None, **kwargs): class SettingsSchema(Schema): + class Meta: + unknown = EXCLUDE + # Settings defined in config files always_create_integration_pull_requests = fields.Bool( required=False, load_default=True) diff --git a/charts/bert-e/templates/secrets.yaml b/charts/bert-e/templates/secrets.yaml index 0e0a16d3..22ed8a71 100644 --- a/charts/bert-e/templates/secrets.yaml +++ b/charts/bert-e/templates/secrets.yaml @@ -10,7 +10,7 @@ metadata: heritage: "{{ .Release.Service }}" type: Opaque data: - BERT_E_GITHOST_PWD: {{ .Values.bertE.robot.password | b64enc | quote }} + BERT_E_ROBOT_PASSWORD: {{ .Values.bertE.robot.password | b64enc | quote }} WEBHOOK_LOGIN: {{ .Values.bertE.webhook.username | b64enc | quote }} WEBHOOK_PWD: {{ .Values.bertE.webhook.password | b64enc | quote }} BERT_E_JIRA_TOKEN: {{ .Values.bertE.jira.token | b64enc | quote }} diff --git a/manifests/base/secrets.env b/manifests/base/secrets.env index 06ee6893..d5950a60 100644 --- a/manifests/base/secrets.env +++ b/manifests/base/secrets.env @@ -1,6 +1,6 @@ BERT_E_CLIENT_SECRET=vault:secret/data/bert-e#BERT_E_GITHUB_OAUTH_CLIENT_SECRET BERT_E_CLIENT_ID=vault:secret/data/bert-e#BERT_E_GITHUB_OAUTH_CLIENT_ID -BERT_E_GITHOST_PWD=vault:secret/data/bert-e#BERT_E_GITHUB_TOKEN +BERT_E_ROBOT_PASSWORD=vault:secret/data/bert-e#BERT_E_GITHUB_TOKEN BERT_E_GITHUB_TOKEN=vault:secret/data/bert-e#BERT_E_GITHUB_TOKEN BERT_E_JIRA_TOKEN=vault:secret/data/bert-e#BERT_E_JIRA_TOKEN BERT_E_JIRA_USER=vault:secret/data/bert-e#BERT_E_JIRA_USERNAME diff --git a/settings.sample.yml b/settings.sample.yml index 4ad03481..16581a5c 100644 --- a/settings.sample.yml +++ b/settings.sample.yml @@ -5,20 +5,20 @@ frontend_url: https://bert-e.mydomain.com/bitbucket/my_company/my_repository/ber # repository_host [MANDATORY]: # The git hosting provider for the repository. Either bitbucket or github. -repository_host: bitbucket +repository_host: github # repository_owner [MANDATORY]: # The name of the owner of the Bitbucket/GitHub repository to work on # # (a.k.a. Bitbucket team or GitHub organization) # -repository_owner: my_company +repository_owner: scality # repository_slug [MANDATORY]: # The slug of the Bitbucket/GitHub repository to work on # -repository_slug: my_repository +repository_slug: bert-e # robot [MANDATORY]: diff --git a/setup.py b/setup.py index 8b20e4e8..b8fb6deb 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import os import pip -from setuptools import setup +from setuptools import setup, find_packages # Besides not advised, @@ -44,16 +44,7 @@ def requires(): url='https://github.com/scality/bert-e', license='Apache', include_package_data=True, - packages=[ - 'bert_e', - 'bert_e.jobs', - 'bert_e.lib', - 'bert_e.bin', - 'bert_e.git_host', - 'bert_e.git_host.github', - 'bert_e.workflow', - 'bert_e.workflow.gitwaterflow', - ], + packages=find_packages(), install_requires=requires(), entry_points={ 'console_scripts': [ diff --git a/tox.ini b/tox.ini index 4c12ce3e..fe883985 100644 --- a/tox.ini +++ b/tox.ini @@ -69,11 +69,13 @@ commands = coverage html [testenv:run] -passenv = BERT_E_* WEBHOOK_* +passenv = + BERT_E_* + WEBHOOK_* setenv = BERT_E_CLIENT_SECRET = {env:BERT_E_CLIENT_SECRET:'bert_e_client_secret'} BERT_E_CLIENT_ID = {env:BERT_E_CLIENT_ID:'bert_e_client_id'} - BERT_E_GITHOST_PWD = {env:GITHUB_TOKEN} + BERT_E_ROBOT_PASSWORD = {env:GITHUB_TOKEN} BERT_E_JIRA_TOKEN = {env:BERT_E_JIRA_TOKEN:'jira_token'} WEBHOOK_LOGIN = {env:WEBHOOK_LOGIN:'webhook'} WEBHOOK_PWD = {env:WEBHOOK_PWD:'webhook'}