diff --git a/.gitignore b/.gitignore index 4910b1a..513e4b4 100755 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,13 @@ +# OA files +mind/**/cache + +# Byte-compiled / optimized / DLL files __pycache__/ -*.pyc -.idea/ +*.py[cod] +*$py.class + +# C extensions +*.so # Distribution / packaging .Python @@ -16,8 +23,82 @@ lib64/ parts/ sdist/ var/ +wheels/ *.egg-info/ .installed.cfg *.egg -mind/**/cache \ No newline at end of file +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/HACKING.rst b/HACKING.rst new file mode 100644 index 0000000..61ac4e4 --- /dev/null +++ b/HACKING.rst @@ -0,0 +1,6 @@ +## Speech Recognizer + + +Generate an fsg file from jsgf + + ```sphinx_jsgf2fsg < conf.jsgf_file > conf.fsg_file``` \ No newline at end of file diff --git a/README.rst b/README.rst index 93ba201..5b33805 100755 --- a/README.rst +++ b/README.rst @@ -1,14 +1,14 @@ Open Assistant ============= -Open Assistant is an evolving open source artificial intelligence agent able +Open Assistant is an evolving open source artificial intelligence agent able to interact in basic conversation and automate an increasing number of tasks. -Maintained by the `Open Assistant `__ -working group lead by `Andrew Vavrek `__, this software -is an extension of `Blather `__ -by `Jezra `__, `Kaylee `__ -by `Clayton G. Hobbs `__, and includes work +Maintained by the `Open Assistant `__ +working group lead by `Andrew Vavrek `__, this software +is an extension of `Blather `__ +by `Jezra `__, `Kaylee `__ +by `Clayton G. Hobbs `__, and includes work done by `Jonathan Kulp `__. @@ -30,15 +30,12 @@ Useful Tools * aplay - console audio player * plaympeg - console mp3 player * projectm - visualizations responsive to sound -* wmctrl - window manager control. opening, closing, resize, switch windows. +* wmctrl - window manager control. opening, closing, resize, switch windows. * xdotool - command line x automation tool * xvkbd - virtual keyboard for x -Running OpenAssistant +Running Open Assistant --------------------- -* The latest documentation can be found on our wiki at http://openassistant.org/wiki/ - -* Install dependencies and tools. Please see http://openassistant.org/wiki/doku.php?id=installation * Download and unpack the latest ``openassistant-master.zip`` package. @@ -54,41 +51,6 @@ Running OpenAssistant * To change assistant commands and language, edit ``conf/commands.json``. Exit and relaunch ``run.sh``. -* For usage instructions, check out the `Open Assistant Wiki `_. - -* For help, you can receive support in the `Open Assistant Forum `_. - - -Next Steps ----------- - -* Port Open Assistant to multiple Linux distributions, beginning with Ubuntu - -* Enable dynamic voice and instant name changes via spoken commands - -* Configure syntax and actions via spoken commands - -* Install internal language model translation - -* Improve speech recognition and synthesis - -* Long-term memory & machine learning - -* Web scraping & information analysis - -* Establish multiple default 'personalities' and plug-in functions - -* Port Open Assistant to all operating systems and devices - -* Galactic Exploration! - - -Join Us! --------- - -Join our development working group at: http://www.openassistant.org -Developers: here's a quick start guide: http://openassistant.org/wiki/doku.php?id=developers - Open Assistant Fork ================== diff --git a/core/util/config.py b/core/util/config.py index 33a9cf6..3d961e8 100644 --- a/core/util/config.py +++ b/core/util/config.py @@ -10,7 +10,7 @@ class Config: def __init__(self, path=None, **opts): logger.info("Loading Mind: {path}".format(path=path)) - + # DIRECTORIES self.cache_dir = os.path.join(path, 'cache') self.conf_dir = os.path.join(path, 'conf') @@ -25,9 +25,13 @@ def __init__(self, path=None, **opts): self._make_dir(self.conf_dir) self._make_dir(self.cache_dir) - + self.options = self._read_options_file() + self.options.update(opts) + logger.info("Options: {}".format(self.options)) + self.commands = self._read_commands_file() + logger.info("Command Count: {}".format(len(self.commands))) def _make_dir(self, directory): @@ -37,21 +41,23 @@ def _make_dir(self, directory): def _read_options_file(self): try: + logger.debug("Reading options from {}".format(self.opt_file)) with open(self.opt_file, 'r') as f: _options = json.load(f) return _options except FileNotFoundError: # MAKE AN EMPTY OPTIONS NAMESPACE - logger.warn("Error loading options file: {path}".format(path=self.opt_file)) + logger.warn("Error reading options file: {path}".format(path=self.opt_file)) return {} def _read_commands_file(self): try: + logger.debug("Reading commands from {}".format(self.cmd_file)) with open(self.cmd_file, 'r') as f: _cmds = json.load(f) return _cmds except FileNotFoundError: - # MAKE AN EMPTY OPTIONS NAMESPACE - logger.warn("Error loading commands file: {path}".format(path=self.cmd_file)) - return {} \ No newline at end of file + # MAKE AN EMPTY COMMANDS NAMESPACE + logger.warn("Error reading commands file: {path}".format(path=self.cmd_file)) + return {} diff --git a/core/util/db.py b/core/util/db.py new file mode 100644 index 0000000..e68f569 --- /dev/null +++ b/core/util/db.py @@ -0,0 +1,47 @@ +import sqlite3 +import json + +#from importlib import reload + + +class DB: + def __init__(self, path=None): + self.path = ":memory:" if path is None else path + self.db = sqlite3.connect(self.path) + + def create_schema(self): + self.db.execute("CREATE TABLE IF NOT EXISTS Prompt (Prompt TEXT)") + self.db.execute("CREATE TABLE IF NOT EXISTS Command (Command TEXT)") + self.db.execute("CREATE TABLE IF NOT EXISTS PromptCommand (PromptID INT, CommandID INT)") + self.db.commit() + + def add_action(self, prompt, command): + p = self.db.execute("SELECT rowid FROM Prompt WHERE Prompt = ?", (prompt,)).fetchone() + if p is None: + prompt_id = self.db.execute("INSERT INTO Prompt (Prompt) VALUES (?)", (prompt,)).lastrowid + else: + prompt_id = p[0] + + c = self.db.execute("SELECT Command FROM Command WHERE Command = ?", (command,)).fetchone() + if c is None: + command_id = self.db.execute("INSERT INTO Command (Command) VALUES (?)", (command,)).lastrowid + else: + command_id = c[0] + + action_id = self.db.execute("SELECT PromptID, CommandID FROM PromptCommand WHERE PromptID=? AND CommandID=?", (prompt_id, command_id)).fetchone() + if action_id is None: + self.db.execute("INSERT INTO PromptCommand (PromptID, CommandID) VALUES (?, ?)", (prompt_id, command_id)) + + self.db.commit() + + def get_action(self, prompt): + action = self.db.execute("SELECT Command FROM Command INNER JOIN PromptCommand ON PromptCommand.CommandID = Command.rowid INNER JOIN Prompt ON Prompt.rowid = PromptCommand.rowid WHERE Prompt = ?", (prompt,)).fetchone() + if action is not None: + return action[0] + + def get_prompts(self): + for prompt in self.db.execute("SELECT Prompt FROM Prompt"): + yield prompt[0] + + def load_commands(self, path): + pass \ No newline at end of file diff --git a/modules/language/__init__.py b/modules/language/__init__.py index 155163b..b4c7a79 100644 --- a/modules/language/__init__.py +++ b/modules/language/__init__.py @@ -9,9 +9,9 @@ #from .hasher import Hasher - NET_TEST_SERVER = "http://www.speech.cs.cmu.edu" + class LanguageUpdater: """ Handles updating the language using the online lmtool. @@ -25,7 +25,7 @@ class LanguageUpdater: def __init__(self, config): self.config = config self.create_strings_file() - + #self.hasher = Hasher(config) # def update_language_if_changed(self): @@ -52,15 +52,55 @@ def __init__(self, config): # self.new_hash = hasher.hexdigest() # # return self.new_hash != self.stored_hash - + + # def create_strings_file(path, source={}): def create_strings_file(self): # Open Strings File + # with open(path, 'w+') as strings: with open(self.config.strings_file, 'w') as strings: # Add Command Words To The Corpus + # for cmd in source: + # strings.write(cmd.strip().replace('%d', '') + "\n") for voice_cmd in sorted(self.config.commands.keys()): strings.write(voice_cmd.strip().replace('%d', '') + "\n") + # def create_sphinx_files(source, lm_path, dic_path): + # """Update the language using the online lmtool""" + # logger.debug("\x1b[32mUpdating Language\x1b[0m") + # + # host = 'http://www.speech.cs.cmu.edu' + # url = host + '/cgi-bin/tools/lmtool/run' + # + # # SUBMIT THE CORPUS TO THE LMTOOL + # response_text = "" + # with open(source, 'rb') as corpus: + # files = {'corpus': corpus} + # values = {'formtype': 'simple'} + # + # r = requests.post(url, files=files, data=values) + # response_text = r.text + # + # # PARSE RESPONSE TO GET URLS OF THE FILES WE NEED + # path_re = r'.*Index of (.*?).*' + # number_re = r'.*TAR([0-9]*?)\.tgz.*' + # for line in response_text.split('\n'): + # # ERROR RESPONSE + # if "[_ERRO_]" in line: + # return 1 + # # IF WE FOUND THE DIRECTORY, KEEP IT AND DON'T BREAK + # if re.search(path_re, line): + # path = host + re.sub(path_re, r'\1', line) + # # IF WE FOUND THE NUMBER, KEEP IT AND BREAK + # elif re.search(number_re, line): + # number = re.sub(number_re, r'\1', line) + # break + # + # lm_url = path + '/' + number + '.lm' + # dic_url = path + '/' + number + '.dic' + # + # _download_file(lm_url, lm_path) + # _download_file(dic_url, dic_path) def update_language(self): """Update the language using the online lmtool""" @@ -109,4 +149,11 @@ def _download_file(self, url, path): if r.status_code == 200: with open(path, 'wb') as f: for chunk in r: - f.write(chunk) \ No newline at end of file + f.write(chunk) + + # def _download_file(url, dest): + # r = requests.get(url, stream=True) + # if r.status_code == 200: + # with open(dest, 'wb') as f: + # for chunk in r: + # f.write(chunk) diff --git a/modules/speech_recognition/gst.py b/modules/speech_recognition/gst.py index 2e5af4e..e232fa7 100644 --- a/modules/speech_recognition/gst.py +++ b/modules/speech_recognition/gst.py @@ -15,16 +15,17 @@ class Recognizer(GObject.GObject): (GObject.TYPE_STRING,)) } + # def __init__(self, mic=None, dic_file=None, lm_file=None, fsg_file=None): def __init__(self, config): GObject.GObject.__init__(self) - self.commands = {} logger.debug("Initializing Recognizer") + self.commands = {} logger.debug(config) logger.debug(config.options) # Configure Audio Source src = config.options['microphone'] - if src: + if src is not None: #audio_src = 'alsasrc device="hw:{0},0"'.format(src) audio_src = 'autoaudiosrc device="hw:{0},0"'.format(src) else: @@ -37,15 +38,16 @@ def __init__(self, config): ' ! audioresample' + ' ! pocketsphinx {}'.format(' '.join([ '{}={}'.format(opt, val) for opt, val in [ - ('lm', config.lang_file), + ('lm', config.lang_file), ('dict', config.dic_file), - ('fsg', config.fsg_file) + ('fsg', config.fsg_file), + ('hmm', config.hmm_path), ] if val is not None ])) + ' ! appsink sync=false' ) logger.debug(cmd) - + try: self.pipeline = Gst.parse_launch(cmd) except Exception as e: @@ -59,9 +61,11 @@ def __init__(self, config): bus.connect('message::element', self.result) def listen(self): + logger.debug("\x1b[32mListening\x1b[0m") self.pipeline.set_state(Gst.State.PLAYING) def pause(self): + logger.debug("\x1b[31mPaused\x1b[0m") self.pipeline.set_state(Gst.State.PAUSED) def result(self, bus, msg): @@ -74,4 +78,5 @@ def result(self, bus, msg): # If We Have A Final Command, Send It For Processing command = msg_struct.get_string('hypothesis') if command != '' and msg_struct.get_boolean('final')[1]: - self.emit("finished", command) \ No newline at end of file + logger.debug("Heard: {}".format(command)) + self.emit("finished", command) diff --git a/modules/speech_recognition/wsr.py b/modules/speech_recognition/wsr.py new file mode 100644 index 0000000..c32aaf0 --- /dev/null +++ b/modules/speech_recognition/wsr.py @@ -0,0 +1,24 @@ +import logging +logger = logging.getLogger(__name__) + +#class Recognizer: +# pass + +# response = speech.input("Say something, please.") +# speech.say("You said " + response) + +# def callback(phrase, listener): +# if phrase == "goodbye": +# listener.stoplistening() +# speech.say(phrase) +# +# listener = speech.listenforanything(callback) +# while listener.islistening(): +# time.sleep(.5) +#listener.stoplistening() +#speech.listenfor(words, lambda phrase, listener: None) + +# import speech +# https://pypi.python.org/pypi/speech/0.5.2 +# req: pywin32 +# https://sourceforge.net/projects/pywin32/files/pywin32 \ No newline at end of file diff --git a/run.py b/run.py index 6e8f5a8..5b9e64b 100755 --- a/run.py +++ b/run.py @@ -4,23 +4,16 @@ import logging logging.basicConfig(level=logging.CRITICAL) -logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) -from argparse import ArgumentParser, Namespace import os import signal import sys import subprocess -from gi.repository import GObject - -from core import Config, Assistant - -from modules.language import LanguageUpdater -from modules.speech_recognition.gst import Recognizer -#from core.numbers import NumberParser +from argparse import ArgumentParser, Namespace def _parser(args): parser = ArgumentParser() @@ -31,8 +24,7 @@ def _parser(args): parser.add_argument("-p", "--pass-words", action="store_true", dest="pass_words", default=False, - help="Pass the recognized words as arguments to the shell" + - " command") + help="Pass the recognized words as arguments to the shell command") parser.add_argument("-H", "--history", type=int, action="store", dest="history", @@ -43,32 +35,54 @@ def _parser(args): help="Audio input card to use (if other than system default)") parser.add_argument("--valid-sentence-command", type=str, - dest="valid_sentence_command", action='store', + dest="valid_sentence_command", action='store', default=None, help="Command to run when a valid sentence is detected") parser.add_argument("--invalid-sentence-command", type=str, - dest="invalid_sentence_command", action='store', + dest="invalid_sentence_command", action='store', default=None, help="Command to run when an invalid sentence is detected") - + parser.add_argument("-M", "--mind", type=str, dest="mind_dir", action='store', help="Path to mind to use for assistant") + parser.add_argument("-d", "--debug", + action='store_true', dest="debug", default=False, + help="Enable debug-level logging") + + parser.add_argument("-u", "--update", + action='store_true', dest="update", default=False, + help="Update language files online") + return parser.parse_args(args) def recognizer_finished(a, recognizer, text): logger.debug("Agent: {}, Recognier: {}, Text: {}".format(a, recognizer, text)) t = text.lower() - #numt, nums = self.number_parser.parse_all_numbers(t) + + # cmd = a.db.get_action(t) + + # # Is There A Matching Command? + # if cmd is not None: + # # Run The 'valid_sentence_command' If It's Set + # os.system('clear') + # if a.config.options['valid_sentence_command']: + # subprocess.call([a.config.options['valid_sentence_command'], text]) + # # Should We Be Passing Words? + # #os.system('clear') + # if a.config.options['pass_words']: + # cmd += " " + t + # print("\x1b[32m< ? >\x1b[0m {0}".format(t)) + # run_command(a, cmd) + # Is There A Matching Command? if t in a.config.commands: # Run The 'valid_sentence_command' If It's Set os.system('clear') print("Open Assistant: \x1b[32mListening\x1b[0m") if a.config.options['valid_sentence_command']: - subprocess.call(a.config.options['valid_sentence_command'], - shell=True) + subprocess.call([a.config.options['valid_sentence_command'], text], shell=True) cmd = a.config.commands[t] # Should We Be Passing Words? os.system('clear') @@ -78,28 +92,14 @@ def recognizer_finished(a, recognizer, text): print("\x1b[32m< ! >\x1b[0m {0}".format(t)) run_command(a, cmd) log_history(a, text) - #elif numt in self.commands: - # # Run 'valid_sentence_command' Set - # os.system('clear') - # print("Open Assistant: \x1b[32mListening\x1b[0m") - # if self.config.options['valid_sentence_command']: - # subprocess.call(self.config.options['valid_sentence_command'], - # shell=True) - # cmd = self.commands[numt] - # cmd = cmd.format(*nums) - # # Should We Be Passing Words? - # if self.config.options['pass_words']: - # cmd += " " + t - # print("\x1b[32m< ! >\x1b[0m {0}".format(t)) - # self.run_command(cmd) - # self.log_history(text) + else: # Run The Invalid_sentence_command If It's Set + logger.debug("Unrecognized command: {}".format(t)) if a.config.options['invalid_sentence_command']: - subprocess.call(a.config.options['invalid_sentence_command'], - shell=True) + subprocess.call([a.config.options['invalid_sentence_command'], text]) print("\x1b[31m< ? >\x1b[0m {0}".format(t)) - + def log_history(a, text): if a.config.options['history']: @@ -112,7 +112,6 @@ def log_history(a, text): with open(a.config.history_file, 'w') as hfile: for line in a.history: hfile.write(line + '\n') - def run_command(a, cmd): """PRINT COMMAND AND RUN""" @@ -121,7 +120,7 @@ def run_command(a, cmd): subprocess.call(cmd, shell=True) recognizer.listen() - + def process_command(self, command): print(command) if command == "listen": @@ -138,49 +137,93 @@ def process_command(self, command): self.quit() +if __name__ == '__main__': + from gi.repository import GObject + + from core import Config, Assistant -if __name__ == '__main__': - # Parse command-line options, # use `Config` to load mind configuration # command-line overrides config file args = _parser(sys.argv[1:]) + if args.debug: + logging.root.setLevel(logging.DEBUG) logger.debug("Arguments: {args}".format(args=args)) - conf = Config(path=args.mind_dir, **vars(args)) - - + + + # Database Prototyping + # from core.util.db import DB + # db = DB(os.path.join(conf.cache_dir, "db")) + # db.create_schema() + # for prompt, command in conf.commands.items(): + # print("Adding {} -> {}".format(prompt, command)) + # db.add_action(prompt, command) + + + # - # Further patching to ease transition.. + # Pre-Configuration # - + # Configure Language logger.debug("Configuring Module: Language") + + # Language Paths conf.strings_file = os.path.join(conf.cache_dir, "sentences.corpus") conf.dic_file = os.path.join(conf.cache_dir, 'dic') + # conf.lm_file = os.path.join(conf.cache_dir, 'lm') conf.lang_file = os.path.join(conf.cache_dir, 'lm') + #XXX: hard coding this for now, sorry :( + conf.hmm_path = "/usr/local/share/pocketsphinx/model/en-us/en-us" conf.fsg_file = None #os.path.join(conf.cache_dir, 'fsg') - # sphinx_jsgf2fsg < conf.jsgf_file > conf.fsg_file - l = LanguageUpdater(conf) - l.update_language() - + + + # Generate Language Files + if args.update: + from modules.language import LanguageUpdater + + # create_strings_file(conf.strings_file, db.get_prompts()) # conf.commands) + # create_sphinx_files(conf.strings_file, conf.lm_file, conf.dic_file) + + l = LanguageUpdater(conf) + l.update_language() + + + + + # Configure Recognizer logger.debug("Configuring Module: Speech Recognition") + from modules.speech_recognition.gst import Recognizer + + # recognizer = Recognizer(args.microphone, dic_file=conf.dic_file, lm_file=conf.lm_file) recognizer = Recognizer(conf) # - # End patching + # End Pre-Configuration # - + # A configured Assistant a = Assistant(config=conf) - + # a.db = db + + + # + # Post-Configuration + # + recognizer.connect('finished', lambda rec, txt, agent=a: recognizer_finished(agent, rec, txt)) - - + + # + # End Post-Configuration + # + + + # # Questionable dependencies # @@ -201,7 +244,7 @@ def process_command(self, command): # could supplant GObject features #a.run() recognizer.listen() - + # Start Main Loop try: @@ -211,4 +254,3 @@ def process_command(self, command): print(e) main_loop.quit() sys.exit() - diff --git a/run.sh b/run.sh index 0cbf905..81d3e13 100755 --- a/run.sh +++ b/run.sh @@ -15,7 +15,7 @@ export KEYPRESS="xvkbd -xsendevent -secure -text" export TERMINAL="tmux new-window " # Use system speech synthesizer on macOS -if [ "$(uname)" = 'Darwin' ] +if [ "$(uname)" = "Darwin" ] then #Mac OSX export VOICE="say"