diff --git a/Modulefile b/Modulefile index c7f2f97f..6bfbe7a4 100644 --- a/Modulefile +++ b/Modulefile @@ -1,5 +1,5 @@ name 'puppet-python' -version '1.0.0' +version '1.1.4' author 'Sergey Stankevich' license 'Apache License, Version 2.0' diff --git a/README.md b/README.md index a7b00d5d..971bbe87 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,25 @@ [puppet-python](https://github.com/stankevich/puppet-python) ====== -Puppet module for installing and managing python, pip, virtualenv, Gunicorn virtual hosts +Puppet module for installing and managing python, pip, virtualenvs and Gunicorn virtual hosts. + +**Version 1.1.x Notes** + +Version 1.1.x makes several fundamental changes to the core of this module, adding some additional features, improving performance and making operations more robust in general. + +Please note that everal changes have been made in v1.1.x which make manifests incompatible with the previous version. However, modifying your manifests to suit is trivial. Please see the notes below. + +Currently, the changes you need to make are as follows: + +* All pip definitions MUST include the owner field which specifies which user owns the virtualenv that packages will be installed in. Adding this greatly improves performance and efficiency of this module. +* You must explicitly specify pip => true in the python class if you want pip installed. As such, the pip package is now independent of the dev package and so one can exist without the other. + +## Installation + +``` bash +cd /etc/puppet/modules +git clone git://github.com/stankevich/puppet-python.git python +``` ## Usage @@ -9,13 +27,15 @@ Puppet module for installing and managing python, pip, virtualenv, Gunicorn virt Installs and manages python, python-dev, python-virtualenv and Gunicorn. -**version** — Python version to install. Default: system default +**version** - Python version to install. Default: system default + +**pip** - Install python-pip. Default: false -**dev** — Install python-dev. Default: false +**dev** - Install python-dev. Default: false -**virtualenv** — Install python-virtualenv. Default: false +**virtualenv** - Install python-virtualenv. Default: false -**gunicorn** — Install Gunicorn. Default: false +**gunicorn** - Install Gunicorn. Default: false class { 'python': version => 'system', @@ -28,64 +48,94 @@ Installs and manages python, python-dev, python-virtualenv and Gunicorn. Installs and manages packages from pip. -**ensure** — present/absent. Default: present +**ensure** - present/absent. Default: present -**virtualenv** — virtualenv to run pip in. +**virtualenv** - virtualenv to run pip in. Default: system (no virtualenv) -**proxy** — Proxy server to use for outbound connections. Default: none +**url** - URL to install from. Default: none - python::pip { 'flask': - virtualenv => '/var/www/project1', - proxy => 'http://proxy.domain.com:3128', +**owner** - The owner of the virtualenv to ensure that packages are installed with the correct permissions (must be specified). Default: root + +**proxy** - Proxy server to use for outbound connections. Default: none + +**environment** - Additional environment variables required to install the packages. Default: none + + python::pip { 'cx_Oracle': + virtualenv => '/var/www/project1', + owner => 'appuser', + proxy => 'http://proxy.domain.com:3128', + environment => 'ORACLE_HOME=/usr/lib/oracle/11.2/client64', } ### python::requirements Installs and manages Python packages from requirements file. -**virtualenv** — virtualenv to run pip in. Default: system-wide +**virtualenv** - virtualenv to run pip in. Default: system-wide -**proxy** — Proxy server to use for outbound connections. Default: none +**proxy** - Proxy server to use for outbound connections. Default: none + +**owner** - The owner of the virtualenv to ensure that packages are installed with the correct permissions (must be specified). Default: root + +**group** - The group that was used to create the virtualenv. This is used to create the requirements file with correct permissions if it's not present already. python::requirements { '/var/www/project1/requirements.txt': virtualenv => '/var/www/project1', proxy => 'http://proxy.domain.com:3128', + owner => 'appuser', + group => 'apps', } ### python::virtualenv Creates Python virtualenv. -**ensure** — present/absent. Default: present +**ensure** - present/absent. Default: present + +**version** - Python version to use. Default: system default -**version** — Python version to use. Default: system default +**requirements** - Path to pip requirements.txt file. Default: none -**requirements** — Path to pip requirements.txt file. Default: none +**proxy** - Proxy server to use for outbound connections. Default: none -**proxy** — Proxy server to use for outbound connections. Default: none +**systempkgs** - Copy system site-packages into virtualenv. Default: don't + +**distribute** - Include distribute in the virtualenv. Default: true + +**owner** - Specify the owner of this virtualenv + +**group** - Specify the group for this virtualenv + +**index** - Base URL of Python package index. Default: none python::virtualenv { '/var/www/project1': ensure => present, version => 'system', requirements => '/var/www/project1/requirements.txt', proxy => 'http://proxy.domain.com:3128', + systempkgs => true, + distribute => false, + owner => 'appuser', + group => 'apps', } ### python::gunicorn Manages Gunicorn virtual hosts. -**ensure** — present/absent. Default: present +**ensure** - present/absent. Default: present + +**virtualenv** - Run in virtualenv, specify directory. Default: disabled -**virtualenv** — Run in virtualenv, specify directory. Default: disabled +**mode** - Gunicorn mode. wsgi/django. Default: wsgi -**mode** — Gunicorn mode. wsgi/django. Default: wsgi +**dir** - Application directory. -**dir** — Application directory. +**bind** - Bind on: 'HOST', 'HOST:PORT', 'unix:PATH'. Default: unix:/tmp/gunicorn-$name.socket or unix:${virtualenv}/${name}.socket -**bind** — Bind on: 'HOST', 'HOST:PORT', 'unix:PATH'. Default: unix:/tmp/gunicorn-$name.socket or unix:${virtualenv}/${name}.socket +**environment** - Set ENVIRONMENT variable. Default: none -**environment** — Set ENVIRONMENT variable. Default: none +**template** - Which ERB template to use. Default: python/gunicorn.erb python::gunicorn { 'vhost': ensure => present, @@ -94,8 +144,12 @@ Manages Gunicorn virtual hosts. dir => '/var/www/project1/current', bind => 'unix:/tmp/gunicorn.socket', environment => 'prod', + template => 'python/gunicorn.erb', } ## Authors [Sergey Stankevich](https://github.com/stankevich) +[Ashley Penney](https://github.com/apenney) +[Marc Fournier](https://github.com/mfournier) +[Fotis Gimian](https://github.com/fgimian) diff --git a/manifests/config.pp b/manifests/config.pp index 43da22e4..49e52307 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -5,7 +5,6 @@ Class['python::install'] -> Python::Virtualenv <| |> Python::Virtualenv <| |> -> Python::Pip <| |> - Python::Virtualenv <| |> -> Python::Requirements <| |> if $python::gunicorn { Class['python::install'] -> Python::Gunicorn <| |> diff --git a/manifests/gunicorn.pp b/manifests/gunicorn.pp index 2548e95d..13f48720 100644 --- a/manifests/gunicorn.pp +++ b/manifests/gunicorn.pp @@ -25,6 +25,9 @@ # [*environment*] # Set ENVIRONMENT variable. Default: none # +# [*template*] +# Which ERB template to use. Default: python/gunicorn.erb +# # === Examples # # python::gunicorn { 'vhost': @@ -34,11 +37,14 @@ # dir => '/var/www/project1/current', # bind => 'unix:/tmp/gunicorn.socket', # environment => 'prod', +# template => 'python/gunicorn.erb', # } # # === Authors # # Sergey Stankevich +# Ashley Penney +# Marc Fournier # define python::gunicorn ( $ensure = present, @@ -46,8 +52,8 @@ $mode = 'wsgi', $dir = false, $bind = false, - $app_interface = 'wsgi', $environment = false, + $template = 'python/gunicorn.erb', ) { # Parameter validation @@ -60,7 +66,7 @@ mode => '0644', owner => 'root', group => 'root', - content => template('python/gunicorn.erb'), + content => template($template), } } diff --git a/manifests/init.pp b/manifests/init.pp index c7ab10f2..2a9a44c3 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -7,6 +7,9 @@ # [*version*] # Python version to install. Default: system default # +# [*pip*] +# Install python-pip. Default: false +# # [*dev*] # Install python-dev. Default: false # @@ -20,6 +23,7 @@ # # class { 'python': # version => 'system', +# pip => true, # dev => true, # virtualenv => true, # gunicorn => true, @@ -31,13 +35,14 @@ # class python ( $version = 'system', + $pip = false, $dev = false, $virtualenv = false, $gunicorn = false ) { # Module compatibility check - $compatible = [ 'Debian', 'Ubuntu' ] + $compatible = [ 'Debian', 'Ubuntu', 'CentOS', 'RedHat' ] if ! ($::operatingsystem in $compatible) { fail("Module is not compatible with ${::operatingsystem}") } diff --git a/manifests/install.pp b/manifests/install.pp index dec99b8a..9306e3a3 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -5,6 +5,11 @@ default => "python${python::version}", } + $pythondev = $::operatingsystem ? { + /(?i:RedHat|CentOS|Fedora)/ => "${python}-devel", + /(?i:Debian|Ubuntu)/ => "${python}-dev" + } + package { $python: ensure => present } $dev_ensure = $python::dev ? { @@ -12,7 +17,13 @@ default => absent, } - package { "${python}-dev": ensure => $dev_ensure } + $pip_ensure = $python::pip ? { + true => present, + default => absent, + } + + package { $pythondev: ensure => $dev_ensure } + package { 'python-pip': ensure => $pip_ensure } $venv_ensure = $python::virtualenv ? { true => present, diff --git a/manifests/pip.pp b/manifests/pip.pp index 9762c103..ef696599 100644 --- a/manifests/pip.pp +++ b/manifests/pip.pp @@ -10,9 +10,18 @@ # [*virtualenv*] # virtualenv to run pip in. # +# [*url*] +# URL to install from. Default: none +# +# [*owner*] +# The owner of the virtualenv being manipulated. Default: root +# # [*proxy*] # Proxy server to use for outbound connections. Default: none # +# [*environment*] +# Additional environment variables required to install the packages. Default: none +# # === Examples # # python::pip { 'flask': @@ -23,11 +32,16 @@ # === Authors # # Sergey Stankevich +# Fotis Gimian # define python::pip ( - $virtualenv, - $ensure = present, - $proxy = false + $package = '', + $ensure = present, + $virtualenv = 'system', + $url = false, + $owner = 'root', + $proxy = false, + $environment = [] ) { # Parameter validation @@ -35,28 +49,51 @@ fail('python::pip: virtualenv parameter must not be empty') } + if $virtualenv == 'system' and $owner != 'root' { + fail('python::pip: root user must be used when virtualenv is system') + } + + $cwd = $virtualenv ? { + 'system' => '/', + default => "${virtualenv}", + } + + $pip_env = $virtualenv ? { + 'system' => 'pip', + default => "${virtualenv}/bin/pip", + } + $proxy_flag = $proxy ? { false => '', default => "--proxy=${proxy}", } - $grep_regex = $name ? { - /==/ => "^${name}\$", - default => "^${name}==", + $grep_regex = $package ? { + /==/ => "^${package}\$", + default => "^${package}==", + } + + $source = $url ? { + false => $package, + default => "${url}#egg=${package}", } case $ensure { present: { exec { "pip_install_${name}": - command => "${virtualenv}/bin/pip install ${proxy_flag} ${name}", - unless => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}", + command => "$pip_env --log-file ${cwd}/pip.log install ${proxy_flag} ${source}", + unless => "$pip_env freeze | grep -i -e ${grep_regex}", + user => $owner, + environment => $environment, } } default: { exec { "pip_uninstall_${name}": - command => "echo y | ${virtualenv}/bin/pip uninstall ${proxy_flag} ${name}", - onlyif => "${virtualenv}/bin/pip freeze | grep -i -e ${grep_regex}", + command => "echo y | $pip_env uninstall ${proxy_flag} ${package}", + onlyif => "$pip_env freeze | grep -i -e ${grep_regex}", + user => $owner, + environment => $environment, } } } diff --git a/manifests/requirements.pp b/manifests/requirements.pp index b6c9b346..67906fc8 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -4,12 +4,24 @@ # # === Parameters # +# [*requirements*] +# Path to the requirements file. Defaults to the resource name +# # [*virtualenv*] # virtualenv to run pip in. Default: system-wide # +# [*owner*] +# The owner of the virtualenv being manipulated. Default: root +# +# [*group*] +# The group relating to the virtualenv being manipulated. Default: root +# # [*proxy*] # Proxy server to use for outbound connections. Default: none # +# [*environment*] +# Additional environment variables required to install the packages. Default: none +# # === Examples # # python::requirements { '/var/www/project1/requirements.txt': @@ -20,16 +32,29 @@ # === Authors # # Sergey Stankevich +# Ashley Penney +# Fotis Gimian # define python::requirements ( - $virtualenv = 'system', - $proxy = false + $requirements = $name, + $virtualenv = 'system', + $owner = 'root', + $group = 'root', + $proxy = false, + $environment = [] ) { - $requirements = $name + if $virtualenv == 'system' and ($owner != 'root' or $group != 'root') { + fail('python::pip: root user must be used when virtualenv is system') + } + + $cwd = $virtualenv ? { + 'system' => '/', + default => "${virtualenv}", + } $pip_env = $virtualenv ? { - 'system' => '`which pip`', + 'system' => 'pip', default => "${virtualenv}/bin/pip", } @@ -38,31 +63,28 @@ default => "--proxy=${proxy}", } - $req_dir = inline_template('<%= requirements.match(%r!(.+)/.+!)[1] %>') - $req_crc = "${requirements}.sha1" - - file { $requirements: - ensure => present, - mode => '0644', - owner => 'root', - group => 'root', - replace => false, - content => '# Puppet will install and/or update pip packages listed here', - } - - # SHA1 checksum to detect changes - exec { "python_requirements_check_${name}": - command => "sha1sum ${requirements} > ${req_crc}", - unless => "sha1sum -c ${req_crc}", - require => File[$requirements], + # This will ensure multiple python::virtualenv definitions can share the + # the same requirements file. + if !defined(File[$requirements]) { + file { $requirements: + ensure => present, + mode => '0644', + owner => $owner, + group => $group, + audit => content, + replace => false, + content => '# Puppet will install and/or update pip packages listed here', + } } - exec { "python_requirements_update_${name}": - command => "${pip_env} install ${proxy_flag} -Ur ${requirements}", - cwd => $virtualenv, + exec { "python_requirements${name}": + provider => shell, + command => "${pip_env} --log-file ${cwd}/pip.log install ${proxy_flag} -r ${requirements}", refreshonly => true, timeout => 1800, - subscribe => Exec["python_requirements_check_${name}"], + user => $owner, + subscribe => File[$requirements], + environment => $environment, } } diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index f22f4e14..039d0853 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -13,9 +13,27 @@ # [*requirements*] # Path to pip requirements.txt file. Default: none # +# [*systempkgs*] +# Copy system site-packages into virtualenv. Default: don't +# +# [*distribute*] +# Include distribute in the virtualenv. Default: true +# +# [*index*] +# Base URL of Python package index. Default: none (http://pypi.python.org/simple/) +# +# [*owner*] +# The owner of the virtualenv being manipulated. Default: root +# +# [*group*] +# The group relating to the virtualenv being manipulated. Default: root +# # [*proxy*] # Proxy server to use for outbound connections. Default: none # +# [*environment*] +# Additional environment variables required to install the packages. Default: none +# # === Examples # # python::virtualenv { '/var/www/project1': @@ -23,17 +41,28 @@ # version => 'system', # requirements => '/var/www/project1/requirements.txt', # proxy => 'http://proxy.domain.com:3128', +# systempkgs => true, +# index => 'http://www.example.com/simple/' # } # # === Authors # # Sergey Stankevich +# Ashley Penney +# Marc Fournier +# Fotis Gimian # define python::virtualenv ( $ensure = present, $version = 'system', $requirements = false, - $proxy = false + $systempkgs = false, + $distribute = true, + $index = false, + $owner = 'root', + $group = 'root', + $proxy = false, + $environment = [] ) { $venv_dir = $name @@ -55,21 +84,46 @@ default => "&& export http_proxy=${proxy}", } + $system_pkgs_flag = $systempkgs ? { + false => '', + default => '--system-site-packages', + } + + $distribute_pkg = $distribute ? { + true => 'distribute', + default => '', + } + $pypi_index = $index ? { + false => '', + default => "-i ${index}", + } + exec { "python_virtualenv_${venv_dir}": - command => "mkdir -p ${venv_dir} \ - ${proxy_command} \ - && virtualenv -p `which ${python}` ${venv_dir} \ - && ${venv_dir}/bin/pip install ${proxy_flag} --upgrade distribute pip", - creates => $venv_dir, + command => "mkdir -p ${venv_dir} ${proxy_command} && virtualenv ${system_pkgs_flag} ${venv_dir} && ${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --upgrade pip ${distribute_pkg}", + user => $owner, + creates => "${venv_dir}/bin/activate", + path => [ '/bin', '/usr/bin', '/usr/sbin' ], + cwd => "/tmp", + environment => $environment, } if $requirements { - Exec["python_virtualenv_${venv_dir}"] - -> Python::Requirements[$requirements] + exec { "python_requirements_initial_install_${requirements}_${venv_dir}": + command => "${venv_dir}/bin/pip --log-file ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} -r ${requirements}", + refreshonly => true, + timeout => 1800, + user => $owner, + subscribe => Exec["python_virtualenv_${venv_dir}"], + environment => $environment, + } - python::requirements { $requirements: - virtualenv => $venv_dir, - proxy => $proxy, + python::requirements { "${requirements}_${venv_dir}": + requirements => $requirements, + virtualenv => $venv_dir, + proxy => $proxy, + owner => $owner, + group => $group, + require => Exec["python_virtualenv_${venv_dir}"], } } diff --git a/templates/gunicorn.erb b/templates/gunicorn.erb index 2cf3eedd..99481e7b 100644 --- a/templates/gunicorn.erb +++ b/templates/gunicorn.erb @@ -1,35 +1,37 @@ CONFIG = { -<% if mode == 'django' -%> +<% if @mode == 'django' -%> 'mode': 'django', <% else -%> 'mode': 'wsgi', <% end -%> -<% if virtualenv -%> +<% if @virtualenv -%> 'environment': { -<% if environment -%> - 'ENVIRONMENT': '<%= environment %>', +<% if @environment -%> + 'ENVIRONMENT': '<%= @environment %>', <% end -%> - 'PYTHONPATH': '<%= virtualenv %>' + 'PYTHONPATH': '<%= @virtualenv %>' }, <% end -%> - 'working_dir': '<%= dir %>', + 'working_dir': '<%= @dir %>', 'user': 'www-data', 'group': 'www-data', -<% if virtualenv -%> - 'python': '<%= virtualenv %>/bin/python', +<% if @virtualenv -%> + 'python': '<%= @virtualenv %>/bin/python', <% else -%> 'python': '/usr/bin/python', <% end -%> 'args': ( -<% if !virtualenv and !bind -%> - '--bind=unix:/tmp/gunicorn-<%= name %>.socket', -<% elsif virtualenv and !bind -%> - '--bind=unix:<%= virtualenv %>/<%= name %>.socket', +<% if !@virtualenv and !@bind -%> + '--bind=unix:/tmp/gunicorn-<%= @name %>.socket', +<% elsif @virtualenv and !@bind -%> + '--bind=unix:<%= @virtualenv %>/<%= @name %>.socket', <% else -%> - '--bind=<%= bind %>', + '--bind=<%= @bind %>', <% end -%> '--workers=<%= @processorcount.to_i*2 %>', '--timeout=30', +<% if @mode != 'django' -%> 'app:app', +<% end -%> ), } diff --git a/tests/gunicorn.pp b/tests/gunicorn.pp index c081fd64..c32ea1a2 100644 --- a/tests/gunicorn.pp +++ b/tests/gunicorn.pp @@ -11,4 +11,5 @@ dir => '/var/www/project1/current', bind => 'unix:/tmp/gunicorn.socket', environment => 'prod', + template => 'python/gunicorn.erb', } diff --git a/tests/requirements.pp b/tests/requirements.pp index a57093f0..9a954fa2 100644 --- a/tests/requirements.pp +++ b/tests/requirements.pp @@ -8,3 +8,8 @@ virtualenv => '/var/www/project1', proxy => 'http://proxy.domain.com:3128', } + +python::requirements { '/var/www/project1/requirements.txt': + virtualenv => 'system', + proxy => 'http://proxy.domain.com:3128', +} diff --git a/tests/virtualenv.pp b/tests/virtualenv.pp index 190492c7..da5ab57c 100644 --- a/tests/virtualenv.pp +++ b/tests/virtualenv.pp @@ -9,4 +9,5 @@ version => 'system', requirements => '/var/www/project1/requirements.txt', proxy => 'http://proxy.domain.com:3128', + systempkgs => true, }