Merge branch '1.0.0-rc2' into master

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "core/webui/public"]
 	path = core/webui/public
 	url = https://github.com/project-alice-assistant/webui.git
-    branch = master
+	branch = master

configTemplate.json

@@ -17,7 +17,15 @@
 		"dataType"    : "boolean",
 		"isSensitive" : false,
 		"description" : "Displays the current system usage on the interface",
-		"category"    : "system"
+		"category"    : "system",
+	  	"onUpdate"    : "WebUIManager.toggleSystemUsage"
+	},
+	"enableSSL"       : {
+	  "defaultValue": false,
+	  "dataType"    : "boolean",
+	  "isSensitive" : false,
+	  "description" : "Enables SSL for both the UI and API",
+	  "category"    : "system"
 	},
 	"delegateNluTraining"     : {
 		"defaultValue": false,
@@ -435,7 +443,7 @@
 			"IBM Watson"    : "watson"
 		},
 		"description" : "The Tts to use. Can't use an online Tts if you have set keepTTSOffline!",
-		"onUpdate"    : "reloadTTS",
+		"onUpdate"    : "reloadTTSManager",
 		"category"    : "tts"
 	},
 	"ttsFallback"             : {
@@ -450,15 +458,15 @@
 			"IBM Watson"    : "watson"
 		},
 		"description" : "The Tts to use in case the default Tts becomes unavailable.",
-		"onUpdate"    : "reloadTTS",
+		"onUpdate"    : "reloadTTSManager",
 		"category"    : "tts"
 	},
 	"ttsLanguage"             : {
 		"defaultValue": "en-US",
 		"dataType"    : "string",
 		"isSensitive" : false,
 		"description" : "Language for the Tts to use",
-		"onUpdate"    : "reloadTTS",
+		"onUpdate"    : "reloadTTSManager",
 		"category"    : "tts"
 	},
 	"ttsType"                 : {
@@ -470,7 +478,7 @@
 			"female"
 		],
 		"description" : "Choose the voice gender you want",
-		"onUpdate"    : "reloadTTS",
+		"onUpdate"    : "reloadTTSManager",
 		"category"    : "tts"
 	},
 	"ttsNeural"             : {
@@ -485,7 +493,7 @@
 		"dataType"    : "string",
 		"isSensitive" : false,
 		"description" : "The voice the Tts should use",
-		"onUpdate"    : "reloadTTS",
+		"onUpdate"    : "reloadTTSManager",
 		"category"    : "tts",
 		"display"     : "hidden"
 	},

core/Initializer.py

@@ -36,6 +36,7 @@
 ASOUND = '/etc/asound.conf'
 TEMP = Path('/tmp/service')
 ALLOWED_LANGUAGES = {'en', 'de', 'fr', 'it', 'pt', 'pl'}
+FALLBACK_ASR = 'coqui'
 
 
 class InitDict(dict):
@@ -46,40 +47,46 @@ def __init__(self, default: dict):
 
 	def __getitem__(self, item):
 		try:
-			item = super().__getitem__(item)
-			if not item:
+			value = super().__getitem__(item)
+			if value is None:
 				raise Exception
-			return item
+			return value
 		except:
 			print(f'Missing key **{item}** in provided yaml file.')
 			return ''
 
 
-class SimpleLogger:
+class SimpleLogger(object):
 
 	def __init__(self, prepend: str = None):
-		self._prepend = f'[{prepend}]\t '
+		self._prepend = f'[{prepend}]'
 		self._logger = logging.getLogger('ProjectAlice')
 
 
 	def logInfo(self, text: str):
-		self._logger.info(f'{self._prepend} {text}')
+		self._logger.info(f'{self.spacer(text)}')
 
 
 	def logWarning(self, text: str):
-		self._logger.warning(f'{self._prepend} {text}')
+		self._logger.warning(f'{self.spacer(text)}')
 
 
 	def logError(self, text: str):
-		self._logger.error(f'{self._prepend} {text}')
+		self._logger.error(f'{self.spacer(text)}')
 
 
 	def logFatal(self, text: str):
-		self._logger.fatal(f'{self._prepend} {text}')
+		self._logger.fatal(f'{self.spacer(text)}')
 		exit(1)
 
 
-class PreInit:
+	def spacer(self, msg: str) -> str:
+		space = ''.join([' ' for _ in range(35 - len(self._prepend) + 1)])
+		msg = f'{self._prepend}{space}{msg}'
+		return msg
+
+
+class PreInit(object):
 	"""
 	Pre init checks and makes sure vital stuff is installed and running. Not much, but internet, venv and so on
 	Pre init is meant to run on the system python and not on the venv
@@ -209,7 +216,7 @@ def doUpdates(self):
 		updateSource = self.getUpdateSource(updateChannel)
 		# Update our system and sources
 		subprocess.run(['sudo', 'apt-get', 'update'])
-		#subprocess.run(['sudo', 'apt-get', 'dist-upgrade', '-y'])
+		subprocess.run(['sudo', 'apt-get', 'dist-upgrade', '-y'])
 		subprocess.run(['sudo', 'apt', 'autoremove', '-y'])
 		subprocess.run(['git', 'clean', '-df'])
 		subprocess.run(['git', 'stash'])
@@ -316,7 +323,7 @@ def checkVenv(self) -> bool:
 		if not Path('venv').exists():
 			self._logger.logInfo('Not running with venv, I need to create it')
 			subprocess.run(['sudo', 'apt-get', 'install', 'python3-dev', 'python3-pip', 'python3-venv', 'python3-wheel', '-y'])
-			subprocess.run(['python3', '-m', 'venv', 'venv'])
+			subprocess.run(['python3.7', '-m', 'venv', 'venv'])
 			self.updateVenv()
 			self._logger.logInfo('Installed virtual environement, restarting...')
 			return False
@@ -331,7 +338,7 @@ def checkVenv(self) -> bool:
 	def updateVenv(self):
 		subprocess.run([self.PIP, 'uninstall', '-y', '-r', str(Path(self.rootDir, 'pipuninstalls.txt'))])
 		subprocess.run([self.PIP, 'install', 'wheel'])
-		subprocess.run([self.PIP, 'install', '-r', str(Path(self.rootDir, 'requirements.txt'))])
+		subprocess.run([self.PIP, 'install', '-r', str(Path(self.rootDir, 'requirements.txt')), '--upgrade', '--no-cache-dir'])
 
 
 	@staticmethod
@@ -355,7 +362,7 @@ def setServiceFileTo(pointer: str):
 		time.sleep(1)
 
 
-class Initializer:
+class Initializer(object):
 	PIP = './venv/bin/pip'
 
 
@@ -396,7 +403,7 @@ def initProjectAlice(self) -> bool:  # NOSONAR
 				self._logger.logFatal('Unfortunately it won\'t be possible, config sample is not existing')
 				return False
 
-			self._confsFile.write_text(json.dumps(self._confsSample.read_text(), indent='\t'))
+			self._confsFile.write_text(self._confsSample.read_text())
 
 		try:
 			confs = json.loads(self._confsFile.read_text())
@@ -427,6 +434,10 @@ def initProjectAlice(self) -> bool:  # NOSONAR
 		subprocess.run(['sudo', 'systemctl', 'stop', 'mosquitto'])
 		subprocess.run('sudo sed -i -e \'s/persistence true/persistence false/\' /etc/mosquitto/mosquitto.conf'.split())
 		subprocess.run(['sudo', 'rm', '/var/lib/mosquitto/mosquitto.db'])
+		subprocess.run(['sudo', 'systemctl', 'start', 'mosquitto'])
+
+		subprocess.run(['sudo', 'systemctl', 'stop', 'nginx'])
+		subprocess.run(['sudo', 'systemctl', 'disable', 'nginx'])
 
 		# Now let's dump some values to their respective places
 		# First those that need some checks and self filling in case unspecified
@@ -435,20 +446,20 @@ def initProjectAlice(self) -> bool:  # NOSONAR
 
 		pinCode = initConfs['adminPinCode']
 		try:
-			pin = int(pinCode)
-			if len(str(pin)) != 4:
+			if len(str(pinCode)) != 4:
 				raise Exception
+			int(pinCode)
 		except:
 			self._logger.logFatal('Pin code must be 4 digits')
 
-		confs['adminPinCode'] = int(pinCode)
+		confs['adminPinCode'] = pinCode
 
 		confs['stayCompletelyOffline'] = bool(initConfs['stayCompletelyOffline'] or False)
 		if confs['stayCompletelyOffline']:
 			confs['keepASROffline'] = True
 			confs['keepTTSOffline'] = True
 			confs['skillAutoUpdate'] = False
-			confs['asr'] = 'deepspeech'
+			confs['asr'] = FALLBACK_ASR
 			confs['tts'] = 'pico'
 			confs['awsRegion'] = ''
 			confs['awsAccessKey'] = ''
@@ -462,14 +473,14 @@ def initProjectAlice(self) -> bool:  # NOSONAR
 			confs['awsAccessKey'] = initConfs['awsAccessKey']
 			confs['awsSecretKey'] = initConfs['awsSecretKey']
 
-			confs['asr'] = initConfs['asr'] if initConfs['asr'] in {'pocketsphinx', 'google', 'deepspeech', 'snips', 'coqui'} else 'deepspeech'
+			confs['asr'] = initConfs['asr'] if initConfs['asr'] in {'pocketsphinx', 'google', 'deepspeech', 'snips', 'coqui'} else FALLBACK_ASR
 			if confs['asr'] == 'google' and not initConfs['googleServiceFile']:
-				self._logger.logInfo('You cannot use Google Asr without a google service file, falling back to Deepspeech')
-				confs['asr'] = 'deepspeech'
+				self._logger.logInfo(f'You cannot use Google Asr without a google service file, falling back to {FALLBACK_ASR}')
+				confs['asr'] = FALLBACK_ASR
 
 			if confs['asr'] == 'snips' and confs['activeLanguage'] != 'en':
-				self._logger.logInfo('You can only use Snips Asr for english, falling back to Deepspeech')
-				confs['asr'] = 'deepspeech'
+				self._logger.logInfo(f'You can only use Snips Asr for english, falling back to {FALLBACK_ASR}')
+				confs['asr'] = FALLBACK_ASR
 
 			if initConfs['googleServiceFile']:
 				googleCreds = Path(self._rootDir, 'credentials/googlecredentials.json')
@@ -519,6 +530,7 @@ def initProjectAlice(self) -> bool:  # NOSONAR
 		try:
 			import pkg_resources
 
+			self._logger.logInfo("*** Trying to load SNIPS-NLU.")
 			pkg_resources.require('snips-nlu')
 			subprocess.run(['./venv/bin/snips-nlu', 'download', confs['activeLanguage']])
 		except:
@@ -557,48 +569,50 @@ def initProjectAlice(self) -> bool:  # NOSONAR
 			confs['disableSound'] = False
 			confs['disableCapture'] = False
 
+		hlcDir = Path('/home', getpass.getuser(), 'HermesLedControl')
 		hlcServiceFilePath = Path('/etc/systemd/system/hermesledcontrol.service')
-		hlcConfigFilePath = Path(f'/home/{getpass.getuser()}/hermesLedControl/configuration.yml')
+		hlcDistributedServiceFilePath = hlcDir / 'hermesledcontrol.service'
+		hlcConfigTemplatePath = hlcDir / 'configuration.yml'
 		hlcConfig = dict()
 		if initConfs['useHLC']:
-
-			hlcDir = Path('/home', getpass.getuser(), 'hermesLedControl')
+			self._logger.logInfo("*** Taking care of HLC.")
 
 			if not hlcDir.exists():
-				subprocess.run(['git', 'clone', 'https://github.com/project-alice-assistant/hermesLedControl.git', str(Path('/home', getpass.getuser(), 'hermesLedControl'))])
+				#todo: replace with AliceGit maybe?
+				subprocess.run(['git', 'clone', 'https://github.com/project-alice-assistant/hermesLedControl.git', str(hlcDir)])
 			else:
 				subprocess.run(['git', '-C', hlcDir, 'stash'])
 				subprocess.run(['git', '-C', hlcDir, 'pull'])
 				subprocess.run(['git', '-C', hlcDir, 'stash', 'clear'])
 
 			if hlcServiceFilePath.exists():
 				subprocess.run(['sudo', 'systemctl', 'stop', 'hermesledcontrol'])
+				subprocess.run(['sudo', 'systemctl', 'disable', 'hermesledcontrol'])
 				subprocess.run(['sudo', 'rm', hlcServiceFilePath])
 
-			subprocess.run(['python3', '-m', 'venv', f'/home/{getpass.getuser()}/hermesLedControl/venv'])
-
-			reqs = [
-				'RPi.GPIO',
-				'spidev',
-				'gpiozero',
-				'paho-mqtt',
-				'numpy',
-				'pyyaml'
-			]
-			for req in reqs:
-				subprocess.run([f'/home/{getpass.getuser()}/hermesLedControl/venv/bin/pip', 'install', req])
+			subprocess.run(['python3.7', '-m', 'venv', f'{str(hlcDir)}/venv'])
+			subprocess.run([f'{str(hlcDir)}/venv/bin/pip', 'install', '-r', f'{str(hlcDir)}/requirements.txt', '--no-cache-dir'])
 
 			import yaml
 
 			try:
-				hlcConfig = yaml.safe_load(hlcConfigFilePath.read_text())
+				hlcConfig = yaml.safe_load(hlcConfigTemplatePath.read_text())
 			except yaml.YAMLError as e:
-				self._logger.logFatal(f'Failed loading HLC configurations: {e}')
-			else:
-				hlcConfig['engine'] = 'projectalice'
-				hlcConfig['pathToConfig'] = f'/home/{getpass.getuser()}/ProjectAlice/config.json'
-				hlcConfig['pattern'] = 'projectalice'
-				hlcConfig['enableDoA'] = False
+				self._logger.logWarning(f'Failed loading HLC configurations - creating new: {e}')
+				hlcConfig = dict()
+
+			hlcConfig['engine'] = 'projectalice'
+			hlcConfig['pathToConfig'] = f'/home/{getpass.getuser()}/ProjectAlice/config.json'
+			hlcConfig['pattern'] = 'projectalice'
+			hlcConfig['enableDoA'] = False
+
+			serviceFile = hlcDistributedServiceFilePath.read_text()
+			serviceFile = serviceFile.replace('%WORKING_DIR%', f'{str(hlcDir)}')
+			serviceFile = serviceFile.replace('%EXECSTART%', f'WorkingDirectory={str(hlcDir)}/venv/bin/python main.py --hermesLedControlConfig=/home/{getpass.getuser()}/.config/HermesLedControl/configuration.yml')
+			serviceFile = serviceFile.replace('%USER%', f'User={getpass.getuser()}')
+			hlcDistributedServiceFilePath.write_text(serviceFile)
+			subprocess.run(['sudo', 'cp', hlcDistributedServiceFilePath, hlcServiceFilePath])
+
 
 		useFallbackHLC = False
 		if initConfs['installSound']:

core/ProjectAlice.py

@@ -68,7 +68,7 @@ def __init__(self, restartHandler: callable):
 
 	def checkDependencies(self) -> bool:
 		"""
-		Compares .hash files against requirements.txt and sysrrequirement.txt. Updates dependencies if necessary
+		Compares .hash files against requirements.txt and sysrequirement.txt. Updates dependencies if necessary
 		:return: boolean False if the check failed, new deps were installed (reboot maybe? :) )
 		"""
 		HASH_SUFFIX = '.hash'
@@ -173,7 +173,7 @@ def updateProjectAlice(self):
 		self._superManager.stateManager.setState(STATE, newState=StateType.RUNNING)
 
 		self._isUpdating = True
-		req = requests.get(url=f'{constants.GITHUB_API_URL}/ProjectAlice/branches', auth=SuperManager.getInstance().configManager.getGithubAuth())
+		req = requests.get(url=f'{constants.GITHUB_API_URL}/ProjectAlice/branches', auth=SuperManager.getInstance().configManager.githubAuth)
 		if req.status_code != 200:
 			self._logger.logWarning('Failed checking for updates')
 			self._superManager.stateManager.setState(STATE, newState=StateType.ERROR)

core/ProjectAliceExceptions.py

@@ -56,11 +56,25 @@ def __init__(self, clazz: str, funcName: str):
 
 class SkillStartingFailed(ProjectAliceException):
 
-	def __init__(self, skillName: str = '', error: str = ''):
+	def __init__(self, skillName: str, error: str = ''):
 		super().__init__(message=error)
 		self._logger.logWarning(f'[{skillName}] Error starting skill: {error}')
 
-		if skillName:
+		if skillName in SuperManager.getInstance().skillManager.NEEDED_SKILLS:
+			self._logger.logFatal(f'Skill **{skillName}** is required to continue, sorry')
+		else:
+			SuperManager.getInstance().skillManager.deactivateSkill(skillName)
+
+
+class SkillInstanceFailed(ProjectAliceException):
+
+	def __init__(self, skillName: str, error: str = ''):
+		super().__init__(message=error)
+		self._logger.logWarning(f'[{skillName}] Error creating skill instance: {error}')
+
+		if skillName in SuperManager.getInstance().skillManager.NEEDED_SKILLS:
+			self._logger.logFatal(f'Skill **{skillName}** is required to continue, sorry')
+		else:
 			SuperManager.getInstance().skillManager.deactivateSkill(skillName)
 
 

core/asr/ASRManager.py

@@ -116,7 +116,7 @@ def _startASREngine(self, forceAsr=None):
 				self.logFatal("Couldn't start any ASR, going down")
 				return
 
-			self.logWarning(f'Something went wrong starting user ASR, falling back to **{fallback}**: {e}')
+			self.logWarning(f'Something went wrong starting user ASR, falling back to **{fallback}**: {e}', printStack=True)
 			self._startASREngine(forceAsr=fallback)
 
 

core/asr/model/ASRResult.py

@@ -23,7 +23,7 @@
 
 
 @dataclass
-class ASRResult:
+class ASRResult(object):
 	text: str
 	session: DialogSession
 	likelihood: float