Docker deployment for tohtori and some refactoring

ansible.cfg

@@ -0,0 +1,2 @@
+[ssh_connection]
+ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o ForwardAgent=yes

applications.yml

@@ -5,6 +5,10 @@
     - nvm
     - koodiklinikka.fi
     - koodiklinikka.fi-api
+    - tohtori
+  vars_files:
+  - vars/application_secret_vars.yml
+  - vars/ejson_key.yml
   tasks:
     - include: roles/nginx/tasks/nginx_config.yml
   handlers:

base.yml

@@ -2,11 +2,14 @@
 - name: Bootstrap a new server
   hosts: all:!localhost
   sudo: yes
-  gather_facts: no
+  gather_facts: yes
   vars_files:
     - vars/users.yml
+    - roles/koodiklinikka.fi/defaults/main.yml
+    - roles/koodiklinikka.fi-api/defaults/main.yml
   roles:
     - base
+    - docker
     - nginx
   tasks:
     - include: tasks/webuser.yml

group_vars/application.yml

@@ -1,54 +0,0 @@
-$ANSIBLE_VAULT;1.1;AES256
-63653230626236303463623861333830663330393363303231373434643834373539616232393839
-3830653765306366323232396334646634343735663234330a333739333663633462373130666137
-35383331386338376230396562663762653831613039663866656430383662333532383136326462
-3039346364333865360a303135316438666133396662393661366333396137663839326664663330
-61646132626561643230613635373235333637386366313130383331626337316434656661343333
-37333034653636616161363736643132636462356661663038356239313063666163366164303739
-62666536343731623837396536303734663430643961383230396135613432643338323435356538
-35646636323462333166373261653038663335353135303333646232643432356263373064313235
-36353463396237353964656666373638636164316436646130383435656165666361353762383736
-35343866386437653936353534626266306636373464613562336161373564346130663332633963
-33666463356637313562363837303633366334336334336330336266656135353133303532336131
-34353133616636303537333665346531313533356134353865653336386332363464376239313065
-35333133313833376531373837613239333833616463646163336134333432376637613831306364
-65646334366465623336636231363334306538373762396535383561386261356564616162316665
-39633639303735303063616437376634623736366432653837663533326135323666323632343632
-34636531353735616266626431346231663865613764323134323833353962366233383535646634
-35386563303039336434646365636133306531343963353464376637626563666231386432373232
-37666665613731396665333231303862646232343864376432383539333730626438313261326561
-31626361636638316664306330343265353130623865643430313236653435326336386133663266
-33363639336134663837626233623736313061636438653537303364376263613134363865643438
-31646565646136626535393231613736306438613037343964626334623435306538373161363265
-65653339643735633238393264373633656233386532323835303264326133313837653561333661
-38316161643466383735373139636164376333303536393136376166656332626635336237376130
-66613466633038373433383432653962613834313761613939396562353663393339623865663430
-33333630656135353435383135303337656162303665386337396633633363343239333634346638
-33393964333566333739346365383836373164333034633437386535663361633765366538323764
-36663636633333666431623333356365333037376432366530633366376437633638376464383461
-37333831386332323066333732323539363738386263323930373364376539626639383030376364
-39666436633934343938363037313033336535626537393332633139373436313163626636376466
-30336334363166663564343062343838633966326365356535313638353330386330316532386161
-63393332386562633232326264353832613436336162356666346538343337663863396130343233
-65383831666136663135363734343832653033333035386464663336376230613866616135383934
-32646332373639623436633362626231653138366633306431306434393436303134626439613663
-37626132646665643839653761393631663535313330666465623031613164353666343832646530
-62306162333934323836393662336633303561363736393335303733663737303261333162396232
-61323435636238656362363631353339353433653062373965383137646238333362393262346461
-32346435326434303861346463663734626436643964393237386561323362633837653734303937
-32343632636364356661323262393964353434653765353537303630373634386361643934333939
-32313039396634306437663233623666373034303238333438326436303664313639326265313133
-65656562353839373734353166383038313837376161653036646264356436636337396536363235
-36643963643033623331626633306366656436393433616132363264303231366638306439613965
-30643864626163633737316663303731346231333965383838616638326464393337616437346137
-32623339353530353365373831303961623431356237663832306264306234333633363162656431
-32373063376462393066316262616339653461343637396630363661616536663563633462393762
-31343836373733643732396638646261623432366363396366633031333937393733663961333937
-34653935333233663138363731656464326239623437336566333562666335653464633639386433
-34316536613037646534316334656266613738626261626162376633313366303935643563333634
-38363234306631373638316336633661343961656336376361626333646239626665633833376534
-34306165663436376338303731646461353830643066303338643131316635613863396239373532
-34333432303831663638623636626630393731346337353937643864633461313434373962633432
-30666430313639346235323639363034323635383636393532346537663538376165343732623666
-31333833666339323836396562396262613239363636313464653361316435363031343338633330
-36663666393237396233636131626133653461313635616133636135323061366632

roles/base/tasks/deps_for_ansible.yml

@@ -0,0 +1,3 @@
+- name: Install python dependencies for managing apt repositories
+  apt: pkg=python-pycurl
+  tags: [nginx, repo]

roles/base/tasks/github_key.yml

@@ -0,0 +1,4 @@
+---
+- name: add github ssh keys
+  shell: ssh-keyscan github.com >> /etc/ssh/ssh_known_hosts
+  sudo: yes

roles/base/tasks/main.yml

@@ -1,4 +1,5 @@
 ---
+- include: deps_for_ansible.yml tags=base,dependencies
 - include: locale.yml tags=base,locale
 - include: users.yml tags=base,users
 - include: packages.yml tags=base,packages
@@ -7,3 +8,4 @@
 - include: ufw.yml tags=base,ufw
 - include: ntp.yml tags=base,ntp
 - include: timezone.yml tags=base,timezone
+- include: github_key.yml tags=base

roles/base/tasks/packages.yml

@@ -9,3 +9,4 @@
     - git
     - htop
     - wget
+    - ruby2.0

roles/base/tasks/swapfile.yml

@@ -0,0 +1,37 @@
+---
+- name: Write swapfile
+  command: |
+    {% if swapfile_use_dd %}
+    dd if=/dev/zero of=/swapfile bs=1M count={{ swapfile_size }} creates=/swapfile
+    {% else %}
+    fallocate -l {{ swapfile_size }} /swapfile creates=/swapfile
+    {% endif %}
+  register: write_swapfile
+  when: swapfile_size != false
+
+- name: Set swapfile permissions
+  file: path=/swapfile mode=600
+  when: swapfile_size != false
+
+- name: Create swapfile
+  command: mkswap /swapfile
+  register: create_swapfile
+  when: swapfile_size != false and write_swapfile.changed
+
+- name: Enable swapfile
+  command: swapon /swapfile
+  when: swapfile_size != false and create_swapfile.changed
+
+- name: Add swapfile to /etc/fstab
+  lineinfile: dest=/etc/fstab line="/swapfile   none    swap    sw    0   0" state=present
+  when: swapfile_size != false
+
+- name: Configure vm.swappiness
+  lineinfile: dest=/etc/sysctl.conf line="vm.swappiness = {{ swapfile_swappiness }}" regexp="^vm.swappiness[\s]?=" state=present
+  notify: Reload sysctl
+  when: swapfile_swappiness != false
+
+- name: Configure vm.vfs_cache_pressure
+  lineinfile: dest=/etc/sysctl.conf line="vm.vfs_cache_pressure = {{ swapfile_vfs_cache_pressure }}" regexp="^vm.vfs_cache_pressure[\s]?=" state=present
+  notify: Reload sysctl
+  when: swapfile_vfs_cache_pressure != false

roles/base/tasks/users.yml

@@ -5,7 +5,7 @@
 - name: create users
   user: >
     name={{ item.name }}
-    groups=admin
+    groups=admin,docker
     shell=/bin/bash
     password={{ item.password }}
   with_items: users

roles/docker/defaults/main.yml

@@ -0,0 +1,10 @@
+---
+
+# The following help expose a docker port or to add additional options when
+# running docker daemon.  The default is to not use any special options.
+#docker_opts: >
+#  -H unix://
+#  -H tcp://0.0.0.0:2375
+#  --log-level=debug
+docker_opts: ""
+

roles/docker/handlers/main.yml

@@ -0,0 +1,10 @@
+---
+# handlers file for docker.ubuntu
+- name: Start Docker
+  service: name=docker state=started
+
+- name: Reload docker
+  service: name=docker state=reloaded
+
+- name: Restart dockerio
+  service: name=docker state=restarted

roles/docker/tasks/main.yml

@@ -0,0 +1,64 @@
+---
+- name: Install lxc-docker
+  apt:
+    pkg: "linux-image-{{ ansible_kernel }}"
+    state: installed
+
+- name: Add Ubuntu universe repo for pip
+  apt_repository:
+    repo: "deb http://mirrors.digitalocean.com/ubuntu trusty universe"
+    update_cache: yes
+    state: present
+
+- name: Install pip
+  apt:
+    pkg: "{{ item }}"
+    state: installed
+  with_items:
+    - python-dev
+
+- name: install pip
+  shell: easy_install -U pip
+  sudo: yes
+
+- name: Install Docker-py
+  pip:
+    name: docker-py
+
+- name: Make sure apt-transport-https is installed
+  apt:
+    pkg: "apt-transport-https"
+    state: installed
+
+- name: Add Docker repository key
+  apt_key:
+    id: "36A1D7869245C8950F966E92D8576A8BA88D21E9"
+    keyserver: "hkp://keyserver.ubuntu.com:80"
+    state: present
+
+- name: Add Docker repository and update apt cache
+  apt_repository:
+    repo: "deb http://get.docker.io/ubuntu docker main"
+    update_cache: yes
+    state: present
+
+- name: Install lxc-docker
+  apt:
+    pkg: "lxc-docker"
+    state: installed
+
+- name: Set docker daemon options
+  copy:
+    content: "DOCKER_OPTS=\"{{ docker_opts.rstrip('\n') }}\""
+    dest: /etc/default/docker
+    owner: root
+    group: root
+    mode: 0644
+  notify:
+    - Reload docker
+  when: docker_opts != ""
+
+- name: Start docker-lxc
+  service:
+    name: docker
+    state: started

roles/koodiklinikka.fi-api/defaults/main.yml

@@ -3,3 +3,10 @@ koodiklinikka_api_project_name: koodiklinikka.fi-api
 koodiklinikka_api_repository_url: git@github.com:koodiklinikka/koodiklinikka.fi-api.git
 koodiklinikka_api_nodejs_version: v0.10.25
 koodiklinikka_api_port: 9000
+
+nvm_path: /opt/web/nvm
+nvm_user: web
+koodiklinikka_api_nvm_path: /opt/web/nvm
+koodiklinikka_api_nvm_script: /opt/web/nvm/nvm.sh
+koodiklinikka_api_app_path: /opt/web/koodiklinikka.fi-api
+koodiklinikka_api_user: web

roles/koodiklinikka.fi/defaults/main.yml

@@ -2,3 +2,7 @@
 koodiklinikka_project_name: koodiklinikka.fi
 koodiklinikka_client_repo: git@github.com:koodiklinikka/koodiklinikka.fi.git
 koodiklinikka_nodejs_version: v0.10.25
+
+koodiklinikka_app_path: /opt/web/koodiklinikka.fi
+koodiklinikka_nvm_script: /opt/web/nvm/nvm.sh
+koodiklinikka_user: web

roles/nginx/tasks/install_nginx.yml

@@ -1,7 +1,4 @@
 ---
-- name: Install python dependencies for managing apt repositories
-  apt: pkg=python-pycurl
-  tags: [nginx, repo]
 
 - name: Add nginx repository
   apt_repository: repo='deb http://nginx.org/packages/ubuntu/ precise nginx' state=present update_cache=yes

roles/tohtori/defaults/main.yml

@@ -0,0 +1,10 @@
+---
+koodiklinikka_tohtori_project_name: tohtori
+koodiklinikka_tohtori_client_repo: git@github.com:koodiklinikka/tohtori.git
+koodiklinikka_tohtori_user: tohtori
+koodiklinikka_tohtori_app_path: /opt/tohtori/tohtori
+
+koodiklinikka_tohtori_container_name: tohtori_bot
+
+ejson_public: 94f3af35ad208d7bc7e3ddf1f9d181d090e3a1c74f9d56851c4f0f1efb04b571
+

roles/tohtori/tasks/build.yml

@@ -0,0 +1,56 @@
+---
+
+- name: Deploy client from git
+  remote_user: "{{ koodiklinikka_tohtori_user }}"
+  action: >
+    git
+    repo="{{ koodiklinikka_tohtori_client_repo }}"
+    dest="{{ koodiklinikka_tohtori_app_path }}"
+    accept_hostkey=True
+  sudo: no
+  register: gitclone
+
+- name: get  previous container
+  shell: docker images|grep {{ koodiklinikka_tohtori_container_name }}|grep latest| awk '{print $3}'
+  register: current_image_sha
+
+- name: Build {{ koodiklinikka_tohtori_container_name }}
+  shell: ./build_docker /opt
+  args:
+    chdir: "{{ koodiklinikka_tohtori_app_path}}/script"
+  sudo: yes
+  #when: gitclone.changed
+  register: container
+
+- name: tag previous version
+  shell: docker tag -f {{ current_image_sha.stdout }} {{ koodiklinikka_tohtori_container_name }}:old
+  when: container is defined and container.changed and current_image_sha.stdout != ""
+
+- name: get running container
+  shell: docker ps -a|grep {{ koodiklinikka_tohtori_container_name }}|grep latest| awk '{print $1}'
+  register: current_container_id
+
+- name: kill old version
+  shell: docker stop {{ koodiklinikka_tohtori_container_name }}
+  when: container is defined and container.changed and current_container_id != ""
+
+- name: get old containers
+  shell: docker ps -a|grep Exited| awk '{print $1}'
+  register: old_containers
+
+- name: clean old containers
+  shell: docker rm $(docker ps -a|grep Exited| awk '{print $1}')
+  when: old_containers.stdout != ""
+
+- name: Start the container
+  shell: docker run -d --name {{ koodiklinikka_tohtori_container_name }} {{ koodiklinikka_tohtori_container_name }}:latest 
+  when: container is defined and container.changed
+
+- name: get old images
+  shell: docker images | grep "^<none>" | awk '{print $3}'
+  register: old_images
+
+- name: clean untagged (old) images
+  shell: docker rmi $(docker images -a | grep "^<none>" | awk '{print $3}')
+  when: old_images.stdout != ""
+  ignore_errors: yes

roles/tohtori/tasks/deploy_ejson.yml

@@ -0,0 +1,9 @@
+- name: Assures /opt/.ejson dir exists
+  file: path=/opt/.ejson state=directory mode=700
+  
+- name: Deploy ejson private key
+  copy:
+    content: "{{ ejson_private_key }}"
+    dest: /opt/.ejson/{{ ejson_public }}
+  tags: [ejson]
+  sudo: true

roles/tohtori/tasks/main.yml

@@ -0,0 +1,4 @@
+---
+- include: tohtori_user.yml
+- include: deploy_ejson.yml
+- include: build.yml

roles/tohtori/tasks/rollback.yml

@@ -0,0 +1,21 @@
+---
+
+- name: get running container
+  shell: docker ps -a|grep {{ koodiklinikka_tohtori_container_name }}|grep latest| awk '{print $1}'
+  register: current_container_id
+
+- name: kill old version
+  shell: docker stop {{ koodiklinikka_tohtori_container_name }}
+  when: current_container_id != ""
+
+- name: get old containers
+  shell: docker ps -a|grep Exited| awk '{print $1}'
+  register: old_containers
+
+- name: clean old containers
+  shell: docker rm $(docker ps -a|grep Exited| awk '{print $1}')
+  when: old_containers.stdout != ""
+
+- name: Start the container
+  shell: docker run -d --name {{ koodiklinikka_tohtori_container_name }} {{ koodiklinikka_tohtori_container_name }}:old
+

roles/tohtori/tasks/tohtori_user.yml

@@ -0,0 +1,5 @@
+---
+- name: Create tohtori user
+  sudo: true
+  user: name=tohtori home=/opt/tohtori password=$1$U7pTMako$SY19s1PIxdD2NCFgM0LQr0
+

rollback_tohtori.yml

@@ -0,0 +1,7 @@
+---
+- name: Rollback tohtori
+  hosts: application
+  vars_files:
+  - roles/tohtori/defaults/main.yml
+  tasks:
+    - include: roles/tohtori/tasks/rollback.yml

run_ansible

@@ -1,4 +1,24 @@
-#!/bin/sh
+#!/bin/bash
-ANSIBLE_SSH_ARGS="-o ForwardAgent=yes"
+
+usage() { echo "$0 -p apps/base -m test/prod";  }
+
+usage
+
+while getopts ":p:m:" OPTION; do
+  case "${OPTION}" in
+    p) PLAYBOOK=${OPTARG};;
+    m) RUN_MODE=${OPTARG};;
+  esac
+done
+
+if [ "$RUN_MODE" == "prod" ]
+  then
+  RUN_MODE=""
+else
+  RUN_MODE="--check"
+fi
+
+
 PW=`ejson --keydir=$HOME/.ejson decrypt secrets/passwords.ejson|grep ansible_vault|cut -d'"' -f4`
-echo $PW|ansible-playbook applications.yml -i hosts --vault-password-file=/bin/cat 
+echo $PW|ansible-playbook $PLAYBOOK.yml -i hosts --vault-password-file=/bin/cat $RUN_MODE -vvvv
+

vars/application_secret_vars.yml

@@ -0,0 +1,35 @@
+$ANSIBLE_VAULT;1.1;AES256
+63316264336165363333376236383664383465306539393934373663633565303531376234373736
+3361333930646465616535333132386164343537366332350a316237613438356336313638623330
+37393530313436346262303336303532376230626530356630373432386433363632613762353966
+6463666433643365340a646366633363326538656365613434313930383030393839343831313136
+34373736636439303639383532376134666136306532383862313837306462616365386439393566
+65326163393563626235613562616630643862656361326561373462313065396436356131303364
+39376261656238313931643265636366656630343131633535656361376664333966666634323566
+36663938666630633431626662376639313436663335613031366265346333313737353165393161
+63623133363836613861323261396234333262376264656136363538326430626561613636376663
+35666334366361613263316235303966326263383437646136343937373232316431373666623564
+32353435663266656239376338313764663538633238626334343330373330643563313862363431
+38643762356134386463316131336637666437333464656537346330303264386532663363393263
+38346564323862646566656435303665303365366430333166663232303236643039623435663731
+33666230646538393463393237656538383365613162633033343666613762643939613562663737
+32326339623065643831323061343261376232616335633137616636313131626332653831636435
+38643763663263616437613265326664623532376437343034646165616230323033303136353832
+39326339373631323634313065336537363166326361663036656231623632393433373637616266
+66373138613033616135336662636230376338336366353366616231663938316564303063353065
+33353833316364326331613737343963363834303936636365396431333966313831323632643265
+37643563356662663963323865646162613730613231343665616138653466346332633765653633
+65653133306163383530366163386235313261343130363634613564653739653730363036393435
+64303165353136623035353266613233333966313631376665343638353232663030646138666438
+36643231666663306138643436373164623866666362633133326361623366303264356565623866
+34373561633365363933326532613537636364376164393662326635633266333166326235376161
+66336465363862316334653136333561373536643437326465323237363265633839306533666333
+37363933616133316336303233303437616332396164393531623232373536363363653562653739
+38636632336566626436623466383064396433323661316462326531613264386566316164666161
+61306430373639306364646537376434333861343463353138646661346538633965346338366337
+34656234376436393834666465393265306138346163396139383966666239643634323063333939
+64613263343830393437323761346537373263376630313532353562316133393335636630376233
+37623061616638306237343435343739386434653831316530626264303666353963633033323264
+63323536373334636364306533346336333665363130396333666531343162303835643831393539
+65333466363762363664353465316130656163616532336530303736336665363438653230316130
+3031656638623932316330633935623435636432306333333563

vars/ejson_key.yml

@@ -0,0 +1,10 @@
+$ANSIBLE_VAULT;1.1;AES256
+65663663633239623135326665306132393837393164343738393430303238643964363662366435
+6332333736383035316662616231616465646135666337660a613437646162613133623731333837
+37373433313062323534333665386264343338333561323030356165353630356135383433616363
+3730646264313561650a643132613164386438613334323764323262383530633731663866666361
+62366139306330303162313062356362396563353863313364306166643964653434313239376537
+39613936353137323438306466386336643237303439616139363335313466363364633738343138
+30656632343564356339346166356265306632343961663864353331333639303563323135386563
+35353337346266343933363836373632336634663737363162666562313465613738333035366538
+6461