Skip to content

Commit

Permalink
Merge branch 'master' into 7152-lattice-app
Browse files Browse the repository at this point in the history
  • Loading branch information
moellep authored Sep 19, 2024
2 parents 408aa44 + 94851a3 commit a4d36b0
Show file tree
Hide file tree
Showing 9 changed files with 84 additions and 72 deletions.
2 changes: 1 addition & 1 deletion sirepo/package_data/static/html/simulations.html
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
<div style="display: inline-block"><a href data-ng-click="simulations.openItem(item)"><span data-ng-class="{ 'sr-user-item': ! simulations.fileManager.isItemExample(item) }">{{ item.name | simulationName | limitTo: 60 }}</span></a> <span data-sr-tooltip="{{ item.notes }}"><span></div>
<ul class="dropdown-menu">
<li><a href data-ng-click="simulations.openItem(item)"><span class="glyphicon sr-nav-icon" data-ng-class="{'glyphicon-folder-open': item.isFolder, 'glyphicon-open-file': ! item.isFolder}"></span> Open</a></li>
<li data-ng-if="! item.isFolder"><a href data-ng-click="simulations.copyItem(item)"><span class="glyphicon glyphicon-duplicate sr-nav-icon"></span> Open as a New Copy</a></li>
<li data-ng-if="! item.isFolder && simulations.canCreateNewSimulation()"><a href data-ng-click="simulations.copyItem(item)"><span class="glyphicon glyphicon-duplicate sr-nav-icon"></span> Open as a New Copy</a></li>
<li data-ng-if="! simulations.fileManager.isItemExample(item)"><a href data-ng-click="simulations.renameItem(item)"><span class="glyphicon glyphicon-edit sr-nav-icon"></span> Rename</a></li>
<li data-ng-if="! simulations.fileManager.isItemExample(item)"><a href data-ng-click="simulations.moveItem(item)"><span class="glyphicon glyphicon-arrow-right sr-nav-icon"></span> Move</a></li>
<li data-ng-if="item.canExport"><a data-ng-href="{{ simulations.exportArchiveUrl(item, 'zip') }}"><span class="glyphicon glyphicon-save-file sr-nav-icon"></span> Export as Zip</a></li>
Expand Down
12 changes: 6 additions & 6 deletions sirepo/package_data/static/js/raydata.js
Original file line number Diff line number Diff line change
Expand Up @@ -1123,7 +1123,7 @@ SIREPO.app.directive('scanDetail', function() {
<div data-ng-if="analysisElapsedTime()"><strong>Analysis Elapsed Time:</strong> {{ analysisElapsedTime() }} seconds</div>
<div>
<div><strong>Current Status: </strong>{{ scan.status }}</div>
<div data-ng-if="! isEmptyObject(latestDetailedStatus)">
<div data-ng-if="latestDetailedStatus">
<strong>Detailed Status:</strong>
<ul>
<li data-ng-repeat="(stepName, stepInfo) in latestDetailedStatus">
Expand All @@ -1146,17 +1146,17 @@ SIREPO.app.directive('scanDetail', function() {
$scope.latestDetailedStatus = null;

function setLatestDetailedStatus() {
$scope.latestDetailedStatus = $scope.scan.detailed_status[Math.max(Object.keys($scope.scan.detailed_status))];
$scope.latestDetailedStatus = null;
if ($scope.scan.detailed_status) {
$scope.latestDetailedStatus = $scope.scan.detailed_status[Math.max(Object.keys($scope.scan.detailed_status))];
}

}

$scope.analysisElapsedTime = () => {
return $scope.scan && $scope.scan.analysis_elapsed_time ? $scope.scan.analysis_elapsed_time : null;
};

$scope.isEmptyObject = (obj) => {
return $.isEmptyObject(obj);
};

$scope.parseTime = (unixTime) => {
return (new Date(unixTime * 1000)).toString();
};
Expand Down
16 changes: 10 additions & 6 deletions sirepo/package_data/static/js/sirepo-components.js
Original file line number Diff line number Diff line change
Expand Up @@ -237,15 +237,16 @@ SIREPO.app.directive('getStarted', function(browserStorage, stringsService) {
<span>
<div class="text-center"><strong>Welcome to Sirepo - ${SIREPO.APP_SCHEMA.appInfo[SIREPO.APP_SCHEMA.simulationType].longName}!</strong></div>
Below are some example ${SIREPO.APP_SCHEMA.strings.simulationDataTypePlural}
and folders containing ${SIREPO.APP_SCHEMA.strings.simulationDataTypePlural}
and folders containing ${SIREPO.APP_SCHEMA.strings.simulationDataTypePlural}.
Click on the ${SIREPO.APP_SCHEMA.strings.simulationDataType}
to open and view the ${SIREPO.APP_SCHEMA.strings.simulationDataType} results.
You can create a new ${SIREPO.APP_SCHEMA.strings.simulationDataType}
by selecting the "${stringsService.newSimulationLabel()}" link above.
<span data-ng-if="SIREPO.APP_SCHEMA.constants.canCreateNewSimulation">You can create a new ${SIREPO.APP_SCHEMA.strings.simulationDataType}
by selecting the "${stringsService.newSimulationLabel()}" link above.</span>
</span>
</div>
`,
controller: function($scope) {
$scope.SIREPO = SIREPO;
const storageKey = 'getStarted';
let isActive = true;

Expand Down Expand Up @@ -2572,7 +2573,7 @@ SIREPO.app.directive('appHeaderRight', function(appDataService, authState, appSt
</li>
</ul>
<ul class="nav navbar-nav" data-ng-show="nav.isActive('simulations')">
<li class="sr-new-simulation-item"><a href data-ng-click="showSimulationModal()"><span
<li data-ng-if="SIREPO.APP_SCHEMA.constants.canCreateNewSimulation" class="sr-new-simulation-item"><a href data-ng-click="showSimulationModal()"><span
class="glyphicon glyphicon-plus sr-small-icon"></span><span class="glyphicon glyphicon-file"></span>
{{ newSimulationLabel() }}</a></li>
<li><a href data-ng-click="showNewFolderModal()"><span class="glyphicon glyphicon-plus sr-small-icon"></span><span
Expand All @@ -2592,7 +2593,7 @@ SIREPO.app.directive('appHeaderRight', function(appDataService, authState, appSt
class="glyphicon glyphicon-exclamation-sign"></span> Report a Bug</a></li>
<li data-help-link="helpUserManualURL" data-title="User Manual" data-icon="list-alt"></li>
<li data-help-link="helpUserForumURL" data-title="User Forum" data-icon="globe"></li>
<li data-ng-if="showLink" data-help-link="helpVideoURL" data-title="Instructional Video" data-icon="film"></li>
<li data-ng-if="SIREPO.APP_SCHEMA.feature_config.show_video_links" data-help-link="helpVideoURL" data-title="Instructional Video" data-icon="film"></li>
</ul>
</li>
</ul>
Expand All @@ -2615,7 +2616,7 @@ SIREPO.app.directive('appHeaderRight', function(appDataService, authState, appSt
controller: function($scope, stringsService) {
$scope.authState = authState;
$scope.slackUri = $scope.authState.slackUri;
$scope.showLink = SIREPO.APP_SCHEMA.feature_config.show_video_links;
$scope.SIREPO = SIREPO;
$scope.modeIsDefault = function () {
return appDataService.isApplicationMode('default');
};
Expand Down Expand Up @@ -3741,6 +3742,9 @@ SIREPO.app.directive('simConversionModal', function(appState, requestSender) {
</div>
`,
controller: function($scope) {
if (SIREPO.APP_SCHEMA.constants.canCreateNewSimulation) {
throw new Error(`SIREPO.APP_SCHEMA.constants.canCreateNewSimulation=${SIREPO.APP_SCHEMA.constants.canCreateNewSimulation} but adding simConversionModal`);
}
$scope.newSimURL = false;
$scope.title = $scope.convMethod == 'create_shadow_simulation' ? 'Shadow' : 'SRW';

Expand Down
6 changes: 5 additions & 1 deletion sirepo/package_data/static/js/sirepo.js
Original file line number Diff line number Diff line change
Expand Up @@ -924,7 +924,7 @@ SIREPO.app.factory('appDataService', function() {
return self.applicationMode;
};
self.canCopy = function() {
return true;
return SIREPO.APP_SCHEMA.constants.canCreateNewSimulation;
};
return self;
});
Expand Down Expand Up @@ -4799,6 +4799,10 @@ SIREPO.app.controller('SimulationsController', function (appState, browserStorag
});
}

self.canCreateNewSimulation = () => {
return SIREPO.APP_SCHEMA.constants.canCreateNewSimulation;
};

self.canDelete = function(item) {
if (item.isFolder) {
return item.children.length === 0;
Expand Down
5 changes: 3 additions & 2 deletions sirepo/package_data/static/json/raydata-schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
}
},
"constants": {
"canCreateNewSimulation": false,
"canDownloadInputFile": false,
"canExportArchive": false,
"canShowDocumentationUrl": false,
Expand Down Expand Up @@ -82,8 +83,8 @@
},
"strings": {
"completionState": "",
"simulationDataType": "notebook",
"simulationDataTypePlural": "notebooks",
"simulationDataType": "analysis",
"simulationDataTypePlural": "analyses",
"typeOfSimulation": "poll"
},
"view": {
Expand Down
1 change: 1 addition & 0 deletions sirepo/package_data/static/json/schema-common.json
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@
}
},
"constants": {
"canCreateNewSimulation": true,
"canDownloadInputFile": true,
"canExportArchive": true,
"canShowDocumentationUrl": true,
Expand Down
9 changes: 9 additions & 0 deletions sirepo/raydata/analysis_driver/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ def get_analysis_pdf_paths(self):
def get_conda_env(self):
raise NotImplementedError("children must implement this method")

def get_detailed_status_file(*args, **kwargs):
return None

def get_notebooks(self, *args, **kwargs):
raise NotImplementedError("children must implement this method")

Expand Down Expand Up @@ -95,6 +98,11 @@ def get_run_log(self):
def has_analysis_pdfs(self):
return len(self.get_analysis_pdf_paths()) > 0

# TODO(e-carlin): There should be a databroker class for each
# beamline and this question should be answered by it.
def is_scan_elegible_for_analysis(self):
return True

def render_papermill_script(self, input_f, output_f):
p = self.get_output_dir().join(_PAPERMILL_SCRIPT)
pkjinja.render_resource(
Expand All @@ -116,6 +124,7 @@ def _get_papermill_args(self, *args, **kwargs):
return []


# TODO(e-carlin): support just passing catalog_name and rduid outsidef of PKDict
def get(incoming):
def _verify_rduid(rduid):
# rduid will be combined with paths throughout the application.
Expand Down
7 changes: 5 additions & 2 deletions sirepo/raydata/analysis_driver/chx.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ class CHX(sirepo.raydata.analysis_driver.AnalysisDriverBase):
def get_conda_env(self):
return _cfg.conda_env

def get_detailed_status_file(self, rduid):
def get_detailed_status_file(self, rduid, *args, **kwargs):
p = self.get_output_dir().join(f"progress_dict_{rduid}.json")
if not p.check():
return PKDict()
return None
d = pkjson.load_any(p)
# The notebooks do json.dump(json.dumps(progress_dict), outfile)
# which double encodes the json object. So, we may
Expand Down Expand Up @@ -56,6 +56,9 @@ def get_output_dir(self):
self.rduid,
)

def is_scan_elegible_for_analysis(self):
return bool(self._scan_metadata.get_start_field("cycle", unchecked=True))

def _get_papermill_args(self, *args, **kwargs):
return [
# Cycle can look like 2024_2 which is converted to int by papermill unless raw_param=True
Expand Down
98 changes: 44 additions & 54 deletions sirepo/raydata/scan_monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,20 +186,15 @@ def set_scan_status(cls, analysis_driver, status, analysis_elapsed_time=None):
r.save()

@classmethod
def statuses_for_scans(cls, catalog_name, rduids):
return (
cls.session.query(cls.rduid, cls.status)
.filter(cls.catalog_name == catalog_name, cls.rduid.in_(rduids))
.all()
)

@classmethod
def analysis_elapsed_time_for_scans(cls, catalog_name, rduids):
return (
cls.session.query(cls.rduid, cls.analysis_elapsed_time)
.filter(cls.catalog_name == catalog_name, cls.rduid.in_(rduids))
.all()
def status_and_elapsed_time(cls, catalog_name, rduid):
r = (
cls.session.query(cls.status, cls.analysis_elapsed_time)
.filter(cls.rduid == rduid)
.one_or_none()
)
if r:
return PKDict(status=r[0], analysis_elapsed_time=r[1])
return PKDict(status=None, analysis_elapsed_time=None)

@classmethod
def _db_upgrade(cls):
Expand Down Expand Up @@ -317,7 +312,7 @@ def _build_search_text(self, text):
},
],
}
elif len(nums):
if len(nums):
return {
"scan_id": {"$in": nums},
}
Expand Down Expand Up @@ -358,32 +353,27 @@ def _search_params(req_data):
)
/ req_data.pageSize
)
l = [
PKDict(rduid=u)
for u in c.search(
_search_params(req_data),
sort=_sort_params(req_data),
limit=req_data.pageSize,
skip=req_data.pageNumber * req_data.pageSize,
)
]
d = PKDict(
_Analysis.statuses_for_scans(
catalog_name=req_data.catalogName, rduids=[s.rduid for s in l]
)
)

e = PKDict(
_Analysis.analysis_elapsed_time_for_scans(
catalog_name=req_data.catalogName, rduids=[s.rduid for s in l]
)
)

for s in l:
s.status = d.get(s.rduid, _AnalysisStatus.NONE)
s.analysis_elapsed_time = e.get(s.rduid, None)
s.detailed_status = _get_detailed_status(req_data.catalogName, s.rduid)
return l, pc
res = []
for u in c.search(
_search_params(req_data),
sort=_sort_params(req_data),
limit=req_data.pageSize,
skip=req_data.pageNumber * req_data.pageSize,
):
# Code after this (ex detailed_status) expects that the
# scan is valid (ex 'cycle' exists in start doc for chx).
# So, don't even show scans to users that aren't elegible.
if sirepo.raydata.analysis_driver.get(
PKDict(catalog_name=req_data.catalogName, rduid=u)
).is_scan_elegible_for_analysis():
res.append(
PKDict(
rduid=u,
detailed_status=_get_detailed_status(req_data.catalogName, u),
**_Analysis.status_and_elapsed_time(req_data.catalogName, u),
)
)
return res, pc

def _request_analysis_output(self, req_data):
return sirepo.raydata.analysis_driver.get(req_data).get_output()
Expand Down Expand Up @@ -602,13 +592,9 @@ def _default_columns(catalog_name):


def _get_detailed_status(catalog_name, rduid):
d = sirepo.raydata.analysis_driver.get(
return sirepo.raydata.analysis_driver.get(
PKDict(catalog_name=catalog_name, rduid=rduid)
)
if hasattr(d, "get_detailed_status_file"):
return d.get_detailed_status_file(rduid)
else:
return None
).get_detailed_status_file(rduid)


async def _init_catalog_monitors():
Expand All @@ -629,7 +615,6 @@ def _monitor_catalog(catalog_name):
# new documents are available.
# But, for now it is easiest to just poll
async def _poll_catalog_for_scans(catalog_name):
# TODO(e-carlin): need to test polling feature
def _collect_new_scans_and_queue(last_known_scan_metadata):
r = [
sirepo.raydata.databroker.get_metadata(s, catalog_name)
Expand Down Expand Up @@ -663,14 +648,17 @@ async def _poll_for_new_scans():
s = _collect_new_scans_and_queue(s)
await pkasyncio.sleep(2)

async def _wait_for_catalog():
while True:
try:
sirepo.raydata.databroker.catalog(catalog_name)
return
except KeyError:
pkdlog(f"no catalog_name={catalog_name}. Retrying...")
await pkasyncio.sleep(15)

pkdlog("catalog_name={}", catalog_name)
c = None
while not c:
try:
c = sirepo.raydata.databroker.catalog(catalog_name)
except KeyError:
pkdlog(f"no catalog_name={catalog_name}. Retrying...")
await pkasyncio.sleep(15)
await _wait_for_catalog()
await _poll_for_new_scans()
raise AssertionError("should never get here")

Expand All @@ -680,6 +668,8 @@ def _queue_for_analysis(scan_metadata):
rduid=scan_metadata.rduid,
catalog_name=scan_metadata.catalog_name,
)
if not sirepo.raydata.analysis_driver.get(s).is_scan_elegible_for_analysis():
return
pkdlog("scan={}", s)
if s not in _SCANS_AWAITING_ANALYSIS:
pkio.unchecked_remove(sirepo.raydata.analysis_driver.get(s).get_output_dir())
Expand Down

0 comments on commit a4d36b0

Please sign in to comment.