diff --git a/.gitignore b/.gitignore index 50ac54ee..bdaf6e12 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ test/temp/ vendor/ composer.phar *.pyc +lib/ansible/playbooks/*.retry +lib/ansible/roles/carlosbuenosvinos.ansistrano-* diff --git a/bin/evolve b/bin/evolve new file mode 100755 index 00000000..24a8df45 --- /dev/null +++ b/bin/evolve @@ -0,0 +1,339 @@ +#!/usr/bin/env python + +from __future__ import print_function +import argparse +import datetime +import getpass +import importlib +import json +import os +import os.path +import pipes +import re +import socket +import subprocess +import sys +import termcolor +import yaml + +class Evolve: + def __init__(self): + raw_args = list(sys.argv) + + # handle arguments & flags + parser = argparse.ArgumentParser(prog='./bin/evolve') + parser.add_argument('stage', help='environment against which to run given action') + parser.add_argument('action', help='action to run against given stage') + parser.add_argument('command', nargs='?', default=None, help='optional (string quoted) subcommand for given action') + parser.add_argument('-v','--verbose', action='count', help='increase verbosity') + parser.add_argument('-e','--extra-vars', action='append', help='provide key=value pairs or JSON document') + self.arguments = parser.parse_args() + self._extra_vars() + + self.announce_verbose('Raw args:', raw_args) + self.announce_verbose('Parsed args:', self.arguments) + + # determine paths + self.working_path = os.getcwd() + self.ansible_path = [ + os.path.abspath(os.path.join(self.working_path, 'lib/ansible')), + os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib/ansible/playbooks')), + ] + self.actions_path = [ + os.path.abspath(os.path.join(self.working_path, 'lib/actions')), + os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib/actions')), + ] + + self.announce_verbose('Paths:', { + 'working_path': self.working_path, + 'ansible_path': self.ansible_path, + 'actions_path': self.actions_path, + }) + + # expose positional arguments for simpler dependency injection + self.stage = self.arguments.stage + self.action = self.arguments.action + self.command = self.arguments.command + + # load ansible group_vars/all and group_vars/{{stage}} (if exists) + self.load_group_vars() + + self.announce('Loading "%s" action in the "%s" stage' % (self.action, self.stage)) + try: + action_module = self.import_action(self.action) + except ImportError as err: + print(termcolor.colored('Unknown action "%s"' % self.action, 'red'), file=sys.stderr) + sys.exit(1) + + # install galaxy roles + # todo: bypass --force unless roles are older than X? + requirements_path = self.playbook_path('galaxy.yml') + if requirements_path: + self.call(['ansible-galaxy', 'install', '-r', requirements_path, '--force']) + else: + raise RuntimeError('Could not find requirements file %s in paths: %s' % (playbook, os.pathsep.join(self.ansible_path))) + + # import and instantiate action + try: + self.init_action(action_module) + except Exception as err: + print(termcolor.colored('The "%s" action failed: %s' % (self.action, err), 'red'), file=sys.stderr) + self.log_action(False, ' '.join(raw_args[1:])) + else: + self.log_action(True, ' '.join(raw_args[1:])) + + def announce(self, message): + ''' + Prints a colored announcement across the terminal. + ''' + print(termcolor.colored("\n! %s" % message, 'green')) + + def announce_verbose(self, label, *args): + if self.arguments.verbose > 3: + print(termcolor.colored(label, 'magenta'), *args) + + def call(self, *args): + ''' + Prints and runs a command, or piped series of commands, inheriting standard input/output and returning after completion. + Throws subprocess.CalledProcessError if returncode was nonzero. + ''' + command = self._normalize_commands(args) + print(termcolor.colored("$ %s\n" % command, 'yellow')) + return subprocess.check_call(command, shell=True) + + def bg_call(self, *args): + ''' + Runs a command, or piped series of commands, ignoring standard input/output and returning a returncode after completion. + ''' + command = self._normalize_commands(args) + self.announce_verbose('Background process:', termcolor.colored("$ %s" % command, 'yellow')) + + proc = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output = proc.communicate() + + result = { 'stdout': output[0], 'returncode': proc.returncode } + self.announce_verbose('Background result:', result) + + return result + + def exec_process(self, command): + executable = command[0] + + if os.access(executable, os.X_OK): + resolved = executable + else: + for path in os.environ['PATH'].split(os.pathsep): + joined = os.path.join(path, executable) + if os.access(joined, os.X_OK): + resolved = joined + break + + if resolved: + os.execv(resolved, command) + else: + raise RuntimeError('Could not find %s in PATH %s' % (executable, os.environ['PATH'])) + + def _normalize_commands(self, args): + ''' + Normalizes and escapes one or more commands (as lists), piping each into the command following it. + ''' + groups = [] + + for command in args: + # remove any None args + command = filter(lambda a: a != None, command) + # quote each arg for shell metacharacters + command = map(pipes.quote, command) + # flatten args to string command + groups.append(' '.join(command)) + + # flatten commands to pipe chain + return ' | '.join(groups) + + def single_input(self, prompt, cb): + ''' + Prompts for a single line of input, validating with a given callback. + ''' + built_prompt = "\n? %s: " % prompt + while True: + result = raw_input(termcolor.colored(built_prompt, 'cyan')) + + try: + if cb(result): + return result + except: + pass + + def multi_input(self, prompt, choices): + ''' + Prompts from a given list of choices. + ''' + built_prompt = termcolor.colored("\n? %s:\n" % prompt, 'cyan') + + for i, choice in enumerate(choices): + choices[i] = choice.lower() + built_prompt += " %s %s\n" % ( + termcolor.colored("[%i]" % (i+1), 'grey', 'on_cyan'), + termcolor.colored(choice.lower(), 'cyan') + ) + + built_prompt += termcolor.colored("> ", 'cyan') + + while True: + result = raw_input(built_prompt) + + try: + test_index = int(result) + if 1 <= test_index <= len(choices): + return choices[test_index-1] + except (ValueError, IndexError, TypeError): + pass + + if result.lower() in choices: + return result.lower() + + def password_input(self, prompt): + ''' + Prompts for a password (which is not shown). + ''' + built_prompt = "\n\a? %s: " % prompt + return getpass.getpass(termcolor.colored(built_prompt, 'white', 'on_blue')) + + def _extra_vars(self): + ''' + Initializes a dictionary of extra variables, to be passed on to ansible. + ''' + extra_vars = {} + if self.arguments.extra_vars: + for pair in self.arguments.extra_vars: + m = re.match(r'^([\w-]+)=([^=]+)$', pair) + if m: + extra_vars[m.group(1)] = m.group(2) + else: + try: + parsed = json.loads(pair) + except ValueError as err: + raise ValueError('%s: %s' % (err, pair)) + extra_vars.update(parsed) + + extra_vars['stage'] = self.arguments.stage + + # infer branch currently checked out (for non prod stages), defaulting to master + if 'branch' not in extra_vars: + extra_vars['branch'] = 'master' + if extra_vars['stage'] != 'production': + result = self.bg_call(['git', 'branch']) + matched = re.search(r'^[*] (\S+)\s', result['stdout'], re.M) + if matched and matched.group(1) != 'master': + extra_vars['branch'] = matched.group(1) + + self.arguments.extra_vars = extra_vars + + def get_extra_vars(self): + ''' + Returns a copy of the extra_vars dictionary. + ''' + return self.arguments.extra_vars.copy() + + def load_group_vars(self): + self.group_vars = {} + vars_files = [ + os.path.join(self.ansible_path[0], 'group_vars/all'), + os.path.join(self.ansible_path[0], 'group_vars/%s' % self.stage), + ] + + for vars_file in vars_files: + if os.path.isfile(vars_file): + f = open(vars_file) + loaded = yaml.safe_load(f) + f.close() + self.group_vars.update(loaded) + + def import_action(self, action): + ''' + Imports a module (adding actions path to system paths if necessary) and returns it. + ''' + for path in self.actions_path: + if path not in sys.path: + sys.path.append(path) + + if '.' in action: + package, module = action.rsplit('.', 1) + module = '.%s' % module + else: + module = action + package = None + + self.announce_verbose('From package %s %s' % ( + termcolor.colored(package, 'yellow'), + termcolor.colored(module, 'green') + )) + + return importlib.import_module(module, package) + + def init_action(self, action_module): + ''' + Initializes an ActionClass from the given module. + ''' + action_module.ActionClass(self) + + def playbook(self, playbook, extra_vars=None, extra_args=None, bg=False): + ''' + Runs an ansible playbook, with optional json encoded extra_vars and optional extra arguments. + ''' + call_args = ['ansible-playbook'] + + # add extra_vars encoded as json + if extra_vars is not None: + call_args += ['-e', json.dumps(extra_vars)] + + # add extra args... + if extra_args is not None: + # append, if a scalar string + if isinstance(extra_args, basestring): + call_args.append(extra_args) + # otherwise add as a list-like + else: + call_args += extra_args + + # append playbook filepath + playbook_path = self.playbook_path(playbook) + if playbook_path: + call_args.append(playbook_path) + else: + raise RuntimeError('Could not find playbook %s in paths: %s' % (playbook, os.pathsep.join(self.ansible_path))) + + if bg: + self.bg_call(call_args) + else: + self.call(call_args) + + def playbook_path(self, playbook): + ''' + Finds the given playbook file in ansible paths, and returns it. + ''' + for path in self.ansible_path: + if os.path.isfile(os.path.join(path, playbook)): + return os.path.join(path, playbook) + + def log_action(self, success, message): + ''' + Writes a log entry to a remote server, via a specialized ansible playbook. + ''' + local_user = getpass.getuser() + local_host = socket.gethostname() + local_time = datetime.datetime.today().strftime('%c') + status = 'Success' if success else 'Failure' + + extra_vars = { + 'stage': self.stage, + 'log_message': '\t'.join([local_time, local_user, message, status]) + } + + try: + self.playbook('log.yml', extra_vars, bg=True) + except subprocess.CalledProcessError as err: + print(termcolor.colored('Failed to log action to remote: %s' % err, 'red'), file=sys.stderr) + +if __name__ == "__main__": + Evolve() diff --git a/bin/mock b/bin/mock index b566066a..b0216f28 100755 --- a/bin/mock +++ b/bin/mock @@ -77,10 +77,9 @@ helpers.testDirectory(outputDir, function(err) { fs.writeFileSync(certPath, cert); } - fs.appendFileSync(outputDir + '/lib/capistrano/deploy.rb', [ + fs.appendFileSync(outputDir + '/lib/ansible/group_vars/all', [ '', - '# Added for testing purposes', - 'set :repo_url, ' + ( + 'ansistrano_git_repo: ' + ( isCI ? '"file:///vagrant/.git"' : '"https://github.com/evolution/wordpress-example.git"' diff --git a/lib/actions/deploy.py b/lib/actions/deploy.py new file mode 100644 index 00000000..9d74c25f --- /dev/null +++ b/lib/actions/deploy.py @@ -0,0 +1,15 @@ +import re +import subprocess +import sys + +class ActionClass: + def __init__(self, evolve): + extra_vars = evolve.get_extra_vars() + extra_vars['deploy_mode'] = 'deploy' + + # infer git remote from currently checked out repo + git_remote = evolve.bg_call(['git', 'config', '--get', 'remote.origin.url']) + extra_vars['git_remote'] = git_remote['stdout'].strip(), + + # kick off ansistrano + evolve.playbook('deploy.yml', extra_vars, ['--user=deploy', '--limit=%s' % evolve.stage]) diff --git a/lib/actions/provision.py b/lib/actions/provision.py new file mode 100644 index 00000000..f1d80722 --- /dev/null +++ b/lib/actions/provision.py @@ -0,0 +1,23 @@ +import re +import subprocess + +class ActionClass: + def __init__(self, evolve): + # TODO: implement `invoke "evolve:prepare_key"` + extra_vars = evolve.get_extra_vars() + + try: + evolve.playbook('provision.yml', extra_vars, '--user=deploy') + except subprocess.CalledProcessError as err: + # only prompt for user reprovisioning if host failed or unreachable + if 'returned non-zero exit status 3' in str(err): + evolve.announce('Unable to provision with SSH publickey for deploy user') + + # connect as intermediate user, and set up deploy user + username = evolve.single_input('user to provision as (root)', lambda u: not u or re.match(r"^[a-z_][a-z0-9_-]*[$]?", u, re.I)) or 'root' + evolve.playbook('user.yml', extra_vars, ['--user=%s' % username, '--ask-pass', '--ask-sudo-pass']) + + # provision as usual + evolve.playbook('provision.yml', extra_vars, '--user=deploy') + else: + raise err diff --git a/lib/actions/rollback.py b/lib/actions/rollback.py new file mode 100644 index 00000000..85f6ba33 --- /dev/null +++ b/lib/actions/rollback.py @@ -0,0 +1,11 @@ +import re +import subprocess +import sys + +class ActionClass: + def __init__(self, evolve): + extra_vars = evolve.get_extra_vars() + extra_vars['deploy_mode'] = 'rollback' + + # kick off ansistrano + evolve.playbook('deploy.yml', extra_vars, ['--user=deploy', '--limit=%s' % evolve.stage]) diff --git a/lib/actions/ssh.py b/lib/actions/ssh.py new file mode 100644 index 00000000..e7e6f3fb --- /dev/null +++ b/lib/actions/ssh.py @@ -0,0 +1,23 @@ +import os +import os.path +import subprocess + +class ActionClass: + def __init__(self, evolve): + extra_vars = evolve.get_extra_vars() + + command = [ + 'ssh', + '-i', + os.path.join(evolve.ansible_path[0], 'files/ssh/id_rsa'), + 'deploy@%s.%s' % (extra_vars['stage'], evolve.group_vars['domain']), + ] + + if evolve.command: + command.append(evolve.command) + try: + evolve.call(command) + except subprocess.CalledProcessError as err: + evolve.announce(err) + else: + evolve.exec_process(command) diff --git a/lib/actions/version.py b/lib/actions/version.py new file mode 100644 index 00000000..32bcbc6e --- /dev/null +++ b/lib/actions/version.py @@ -0,0 +1,17 @@ +import json +import os +import os.path + +class ActionClass: + def __init__(self, evolve): + extra_vars = evolve.get_extra_vars() + + # inject generated version (from group vars) + extra_vars['generated_version'] = evolve.group_vars['evolve_version'] + + # inject installed version (from bower_components) + with open(os.path.join(evolve.working_path, 'bower_components/evolution-wordpress/package.json'), 'r') as package_json: + extra_vars['local_version'] = json.load(package_json)['version'] + + # call playbook to gather and display all three versions + evolve.playbook('version.yml', extra_vars=extra_vars) diff --git a/lib/actions/wp.py b/lib/actions/wp.py new file mode 100644 index 00000000..45508e72 --- /dev/null +++ b/lib/actions/wp.py @@ -0,0 +1,19 @@ + +class ActionClass: + def __init__(self, evolve): + extra_vars = evolve.get_extra_vars() + + if not evolve.command: + raise RuntimeError('wp action requires a command string'); + + ssh_module = evolve.import_action('ssh') + + evolve.command = ' '.join([ + '/usr/local/bin/wp', + evolve.command, + '--path="%s"' % evolve.group_vars['wp_path'], + '--url="http://%s.%s/"' % (extra_vars['stage'], evolve.group_vars['domain']), + '--color', + ]) + + evolve.init_action(ssh_module) diff --git a/lib/ansible/playbooks/deploy.yml b/lib/ansible/playbooks/deploy.yml new file mode 100644 index 00000000..920c659b --- /dev/null +++ b/lib/ansible/playbooks/deploy.yml @@ -0,0 +1,6 @@ +--- +- hosts: "{{ stage }}" + gather_facts: yes + + roles: + - { role: "carlosbuenosvinos.ansistrano-{{ deploy_mode }}" } diff --git a/lib/ansible/playbooks/log.yml b/lib/ansible/playbooks/log.yml new file mode 100644 index 00000000..33c607f4 --- /dev/null +++ b/lib/ansible/playbooks/log.yml @@ -0,0 +1,20 @@ +--- +- hosts: "{{ stage }}" + gather_facts: yes + + vars: + log_dir: "/var/log/evolution" + log_file: "{{ log_dir }}/wordpress.log" + + tasks: + - name: Ensure log directory exists and is owned by remote user + file: path={{ log_dir }} state=directory mode=0644 owner={{ ansible_ssh_user }} + sudo: yes + + - name: Ensure log file exists and is owned by remote user + file: path={{ log_file }} state=touch mode=0644 owner={{ ansible_ssh_user }} + sudo: yes + + - name: Append message to log file + lineinfile: insertafter=EOF dest={{ log_file }} line={{ log_message }} + sudo: yes diff --git a/lib/ansible/playbooks/version.yml b/lib/ansible/playbooks/version.yml new file mode 100644 index 00000000..56fe6a75 --- /dev/null +++ b/lib/ansible/playbooks/version.yml @@ -0,0 +1,17 @@ +--- +- hosts: "{{ stage }}" + + tasks: + - name: Retrieve service version + command: service evolution-wordpress version + register: provisioned_version + sudo: yes + + - debug: + msg: "Last generated with: {{generated_version}}" + + - debug: + msg: "Currently bower installed with: {{local_version}}" + + - debug: + msg: "Stage '{{stage}}' last provisioned with: {{provisioned_version.stdout|regex_replace('\\A[\\s\\S]*?Evolution Wordpress ([\\w.-]+)[\\s\\S]*?\\Z', '\\1')}}" diff --git a/lib/ansistrano/after-update-code-tasks.yml b/lib/ansistrano/after-update-code-tasks.yml new file mode 100644 index 00000000..d700a7af --- /dev/null +++ b/lib/ansistrano/after-update-code-tasks.yml @@ -0,0 +1,5 @@ +--- +- name: Install bower dependencies + command: bower install --config.interactive=false + args: + chdir: "{{ ansistrano_release_path.stdout_lines[0] }}" diff --git a/lib/ansistrano/before-setup-tasks.yml b/lib/ansistrano/before-setup-tasks.yml new file mode 100644 index 00000000..041bf0dd --- /dev/null +++ b/lib/ansistrano/before-setup-tasks.yml @@ -0,0 +1,15 @@ +--- +- name: Use branch fom extra_vars, when explicitly provided + set_fact: + ansistrano_git_branch: "{{ branch }}" + when: branch is defined + +- name: Use git_remote from extra_vars, if the default was not overriden + set_fact: + ansistrano_git_repo: "{{ git_remote }}" + when: ansistrano_git_repo.find('USERNAME/REPO.git') != -1 + +- name: Clear deploy key for https remotes + set_fact: + ansistrano_git_identity_key_path: '' + when: ansistrano_git_repo.startswith('https://') diff --git a/lib/yeoman/index.js b/lib/yeoman/index.js index 308f2c6e..9f09972c 100644 --- a/lib/yeoman/index.js +++ b/lib/yeoman/index.js @@ -700,6 +700,8 @@ var WordpressGenerator = yeoman.generators.Base.extend({ fixPermissions: function() { fs.chmodSync(path.join(this.env.cwd, 'lib', 'ansible', 'files', 'ssh', 'id_rsa'), '600'); + fs.chmodSync(path.join(this.env.cwd, 'bin', 'evolve'), '755'); + fs.chmodSync(path.join(this.env.cwd, 'bin', 'import'), '755'); }, installGems: function() { diff --git a/lib/yeoman/templates/_gitignore b/lib/yeoman/templates/_gitignore index 970ad645..44721816 100644 --- a/lib/yeoman/templates/_gitignore +++ b/lib/yeoman/templates/_gitignore @@ -6,6 +6,7 @@ lib/ansible/files/ssl/ <% } %> # Application files error_log +lib/ansible/*.retry # WordPress files **/wp-content/cache/** diff --git a/lib/yeoman/templates/bin/evolve b/lib/yeoman/templates/bin/evolve new file mode 100644 index 00000000..34744671 --- /dev/null +++ b/lib/yeoman/templates/bin/evolve @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +IMPORTER_PATH="$( dirname "$0" )/../bower_components/evolution-wordpress" +IMPORTER_SCRIPT="$IMPORTER_PATH/bin/evolve" + +chmod +x "$IMPORTER_SCRIPT" + +$IMPORTER_SCRIPT "$@" diff --git a/lib/yeoman/templates/lib/actions/_gitkeep b/lib/yeoman/templates/lib/actions/_gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/lib/yeoman/templates/lib/ansible/galaxy.yml b/lib/yeoman/templates/lib/ansible/galaxy.yml index c93bbb46..f147d672 100644 --- a/lib/yeoman/templates/lib/ansible/galaxy.yml +++ b/lib/yeoman/templates/lib/ansible/galaxy.yml @@ -1 +1,3 @@ +- src: carlosbuenosvinos.ansistrano-deploy +- src: carlosbuenosvinos.ansistrano-rollback <% if (props.datadog) { %>- src: Datadog.datadog<% } %> diff --git a/lib/yeoman/templates/lib/ansible/group_vars/all b/lib/yeoman/templates/lib/ansible/group_vars/all index 59c7517a..e8134464 100644 --- a/lib/yeoman/templates/lib/ansible/group_vars/all +++ b/lib/yeoman/templates/lib/ansible/group_vars/all @@ -35,3 +35,15 @@ mail__postmaster: no-reply@<%= props.domain %> # swap__path: /swapfile # swap__swappiness: 10 # swap__vfs_cache_pressure: 50 + +ansistrano_deploy_to: "/var/www/{{ domain }}/{{ stage }}/{{ branch }}" +ansistrano_shared_paths: ['web/wp-content/uploads'] +ansistrano_keep_releases: 2 +ansistrano_deploy_via: "git" +ansistrano_allow_anonymous_stats: no +ansistrano_git_identity_key_path: "files/ssh/id_rsa" +ansistrano_before_setup_tasks_file: "{{ playbook_dir }}/../../ansistrano/before-setup-tasks.yml" +ansistrano_after_update_code_tasks_file: "{{ playbook_dir }}/../../ansistrano/after-update-code-tasks.yml" +ansistrano_remove_rolled_back: yes + +wp_path: "{{ ansistrano_deploy_to }}/current/web/wp" diff --git a/lib/yeoman/templates/lib/ansible/group_vars/local b/lib/yeoman/templates/lib/ansible/group_vars/local new file mode 100644 index 00000000..9c7fdb82 --- /dev/null +++ b/lib/yeoman/templates/lib/ansible/group_vars/local @@ -0,0 +1,2 @@ +--- +wp_path: "/vagrant/web/wp"