Fabric Python Lib

Fabric al PyCon 5 di FirenzeTaa taa chi trova fabric non lo lascia più…

$ fab taskA taskB

from fabric.api import run, env

env.hosts = ['host1', ‘host2']

def taskA(): run(‘ls')

def taskB(): run('whoami')

$ fab mytask:roles=role1

from fabric.api import env

env.roledefs = { 'web': ['www1', 'www2', 'www3'], 'dns': ['ns1', ‘ns2']

}

def mytask(): run('ls /var/www')

$ fab mytask:roles=role1,exclude_hosts="a;c"

from fabric.api import env, hosts, roles, run

env.roledefs = {'role1': ['b', ‘c']}

@hosts('a', ‘b') @roles(‘role1') def mytask():

run('ls /var/www')

$ fab migrate update

from fabric.api import run, roles

env.roledefs = { 'db': ['db1', 'db2'], 'web': ['web1', 'web2', ‘web3'],

}

@roles(‘db') def migrate():

# Database stuff here. pass

@roles(‘web') def update():

# Code updates here. pass

fab deploy

from fabric.api import run, roles, execute

def deploy(): execute(migrate) execute(update)

migrate on db1 migrate on db2 update on web1 update on web2 update on web3

$ fab deploy:app or $ fab deploy:db

from fabric.api import run, execute, task

from mylib import external_datastore

def do_work(): run("something interesting on a host”)

@task def deploy(lookup_param):

host_list = external_datastore.query(lookup_param) execute(do_work, hosts=host_list)

$ fab set_hosts:app do_work

from fabric.api import run, task from mylib import external_datastore

@task def do_work():

run("something interesting on a host")

@task def set_hosts(lookup_param):

env.hosts = external_datastore.query(lookup_param)

Combining stdout and stderr

run("cmd", pty=False, combine_stderr=True):

run("cmd", pty=False, combine_stderr=False):

run("cmd", pty=True, combine_stderr=False):

$ fab -H host1,host2,host3 runs_in_parallel runs_seriallyfrom fabric.api import *

@parallel def runs_in_parallel():

pass

def runs_serially(): pass

runs_in_parallel on host1, host2, and host3 runs_serially on host1 runs_serially on host2 runs_serially on host3

$ fab -P -z 5 heavy_task

from fabric.api import *

@parallel(pool_size=5) def heavy_task(): # lots of heavy local lifting or lots of IO here

@task(alias=’short’)

from fabric.api import task, run

@task def mytask():

run("a command")

@task(alias=‘dwm’) def deploy_with_migrations():

pass

Submodule deploy.py

@task(default=True) def full_deploy():

pass

$ fab --list Available commands:

deploy deploy.full_deploy deploy.migrate

Class Task

class MyTask(Task): name = "deploy" def run(self, environment, domain="whatever.com"): run("git clone foo") sudo("service apache2 restart")

instance = MyTask()

VS

@task def deploy(environment, domain="whatever.com"):

run("git clone foo") sudo("service apache2 restart")

Colors

from fabric.colors import green print(green("This text is green!"))

fabric.colors.blue(text, bold=False) fabric.colors.cyan(text, bold=False) fabric.colors.green(text, bold=False) fabric.colors.magenta(text, bold=False) fabric.colors.red(text, bold=False) fabric.colors.white(text, bold=False) fabric.colors.yellow(text, bold=False)

Context managersdef mytask():

with cd('/path/to/app'), prefix('workon myvenv'): run('./manage.py syncdb') run('./manage.py loaddata myfixture') with cd('/var/www'): run('ls') # cd /var/www && ls with cd('website1'): run('ls') # cd /var/www/website1 && ls with hide('running', 'stdout', 'stderr'): run('ls /var/www') # Map localhost:6379 on the server to localhost:6379 on the client, # so that the remote 'redis-cli' program ends up speaking to the local # redis-server. with remote_tunnel(6379): run("redis-cli -i")

Contrib

Django Integration Rsync Project Upload Project Console Confirm y/n Files and Directory

So… what is Fabric?

● Deploy ● Manage multiple server ● Clustering ● Multiplatform ● Parallel ● Testing ● SSH Authentication

SSH + Bash Power + Python = Rocks**3 = Fabric

fab release:master

…

django integration

from fabric.contrib import django

django.settings_module(‘myproject.settings')

from django.conf import settings

def dump_production_database(): run(pg_dump -U %s -w %s > /bck/prod-db.sql' % ( settings.DATABASE_USER, settings.DATABASE_NAME ))

fab deploy

@hosts('user@example.com')def deploy():

with cd("/opt/myproject"): run("git pull") run("django-admin.py collectstatic --noinput") run("django-admin.py migrate --noinput") run("/etc/init.d/uwsgi stop || echo 'done'") run("/etc/init.d/uwsgi start")

fab backup_and_publish

@task@hosts('www-data@example.com')def backup_and_publish(): run('''tar cjvf /var/wwwbackups/www.%s.tar.bz2

--exclude=/var/www/download —exclude=/var/www/backup* /var/www''' % today().strftime("%Y%m%d%H%M%S")) run('rsync -avz --checksum --ignore-times

/var/wwwstage/ /var/www') #--delete

fab static_generation

@taskdef static_generation():

execute(remote_generation) local("wget --user=admin --password=rootme --recursive --page-requisites --html-extension --convert-links --restrict-file-names=windows --domains example.com --no-parent http://wwwstage.example.com/ -o dump.log || echo 'Looking for 404 on wwwstage.example.com'") local("cat dump.log | grep -B 2 '404 Not Found' | grep 'http://wwwstage.example.com/' || echo 'OK no 404 found...'")

fab upload_release_note:2.10.0

@task@hosts('install@example.com')def upload_release_note(version):

release_note_file_name = "RELEASE_NOTE_%s.TXT" % version with open(release_note_file_name,"w") as out_file: notes = jira.Query().render(version=version) out_file.write(notes.encode('ascii', 'ignore')) out_file.close() put(release_note_file_name, "/cygdrive/d/%s/" % version)

fab cleanup_nexus:10.0.2-RC2

@taskdef cleanup_nexus(version):

for module in [ "core-api", "core-client-rest", "core-manual", "core-web"]:

local("curl -X DELETE -u user:rootme http://nexus.example.com:8180/nexus/service/local/repositories/releases/content/example/%s/%s/" % (module, version))

LOCK_FILE = “~/.lockfile.release.core.lock"

class Lock(): def __enter__(self): if os.access(os.path.expanduser(LOCK_FILE), os.F_OK): pidfile = open(os.path.expanduser(LOCK_FILE), "r") pidfile.seek(0) old_pid = pidfile.readline() print "There is an already a process running with pid: %s," % old_pid sys.exit(1) pidfile = open(os.path.expanduser(LOCK_FILE), "w") pidfile.write("%s" % os.getpid()) pidfile.close def __exit__(self, type, value, traceback): os.remove(os.path.expanduser(LOCK_FILE))

Locks

fab send_email_candidate:2.12,me@...

@taskdef send_mail_candidate(version, *receivers):

sender = 'development@geniusbytes.com' body = """From: Core Team <noreply@example.com>To: Development <development@example.com>Subject: New Release CANDIDATE %(version)s\nNew Release CANDIDATE %(version)savailable on: * smb://example.com/myproject/%(version)s """ % dict(version=version) try: message = smtplib.SMTP('example.com') message.sendmail(sender, receivers, body) print "Successfully sent email" except smtplib.SMTPException: print "Error: unable to send email"

XML parser@taskdef get_pom_version():

src=os.path.dirname(__file__) pom_file=os.path.abspath(os.path.join(src, 'pom.xml')) from xml.dom.minidom import parse pom = parse(pom_file) version = pom.getElementsByTagName("version")[1].firstChild.nodeValue find = re.compile(r'^\d+.\d+.\d+-([a-zA-Z-]+)\d*-SNAPSHOT$').findall(version) if not find: abort(version + " is not a valid development version") versions = re.compile(r'\d+').findall(version) if len(versions) is 3: versions.append(1) if len(versions) is not 4: abort(version + " is not a valid development version") versions.extend(find) return versions

fab sed_poms_version:2.13

@taskdef sed_poms_version(new_version):

major, minor, patch, rev, rc = get_pom_version()

version = "%s.%s.%s-%s%s-SNAPSHOT" % (major, minor, patch, rc, rev)

local("sed -i '' 's@%s@%s@g' pom.xml */pom.xml" % (version, new_version))

fab create_manuals:2.10

ACTIVATE="source /home/installer/.sphinx_env/bin/activate"@hosts('installer@example.com')def create_manuals(version): with cd(DEPLOY_DIR + "/myproject"): name = "manual-%s-doc" % version run("wget http://nexus.example.com:8180/nexus/service/local/repo_groups/public/content/com/myproject/manual/%s/%s.zip" % (version, name)) with cd(name): run(ACTIVATE + "&& make docs") run("mv target/en/latex/MyProject*.pdf docs/" % version) run("tar cjvf docs/MyCDDS-en-%s.tar.bz2 target/docs/" % version) run("scp -r docs install@example.com:/cygdrive/d/%s/docs" % version)

fab create_installer:2.10 (innosetup)

@hosts('install@example.com')def create_installer(version): name = "installer-%s-app" % version run("wget http://nexus.geniusbytes.com:8180/nexus/service/local/repo_groups/public/content/myproject/installer/%s/%s.zip" % (version, name)) with cd(name): run("tar xjvf /cygdrive/d/%(version)s/MyCDDS-en-%(version)s.tar.bz2" % dict(version=version)) run(ANT) run("mkdir -p /cygdrive/d/%s" % version) run("cp build/*.exe /cygdrive/d/%s/" % version) run("cp build-update/*.exe /cygdrive/d/%s/" % version) run("rm -rf %s" % name)

fab release_core:master,2.10,2.11

@hosts('installer@ecample.com')def release_core(branch, version, next_version):

with cd(CORE_DEPLOY_DIR): run('git clone ssh://i@example.com/srv/git/myproject.git') with cd(CORE_DEPLOY_DIR + "/myproject"): run("git checkout %s" % branch) run("mvn clean install") run("mvn --batch-mode release:clean release:prepare

-DreleaseVersion=%s -DdevelopmentVersion=%s" % (version, next_version))

run("mvn release:perform")

fab release:master

def release(branch): with Lock(): major, minor, patch, rev, rc = check_current_version(branch) if 'RC' != rc: abort("RC not found, not possible release a final version") version = "%s.%s.%s" % (major, minor, patch) next_version = "%s.%s.%s-RC%s-SNAPSHOT" % (major, minor, int(patch)+1, 1) puts("preparing to release %s (next will be %s)" % (version, next_version)) execute(release_core, branch, version, next_version) execute(create_manuals, version) execute(create_installer, version) execute(upload_release_note, version) execute(send_mail_final_release, version, 'me@ex.com', 'dev@ex.com') local("git pull") execute(labels.missing_translations, 'me@ex.com')

fab cluod_remote_control

...

EC2 Testing with 200 micro server

EC2 + Fabric + Funkload

● EC2 use all ubuntu standard AMI ● Fabric as remote control, move files, aggregate. ● Funkload in order to stress an internet application. (not on

EC2)

Performance Testing Architecture

Target

Cloud CTRL

Tester

TargetTarget

FunkLoad

Fabric (nmon+pefmon)

Fabric + EC2

Fabric + EC2

Testing phases

I. Prepare Monitoring II.Prepare Cloud

1. Start Monitoring 2. Start Parallel Testing 3. Collecting Test Results 4. Collecting Perf Results 5. Reporting

Target

Cloud CTRL

Tester

TargetTarget

FunkLoad

Fabric (nmon+pefmon)

Fabric + EC2

Fabric + EC2

Testing Console

fab prepare_monitoring fab prepare_cloud

fab start_monitoring fab start_testing:ciccio,100,5000 fab collecting_test_results:ciccio fab collecting_perf_results:ciccio fab reporting:ciccio