First public release

.gitignore

@@ -0,0 +1,10 @@
+.gradle/
+build/
+logs/
+temp/
+*~
+*.swp
+.classpath
+.project
+.settings
+gradle.properties

CHANGELOG.md

@@ -0,0 +1,3 @@
+# 1.0.0 (2014-09-15)
+
+First public release

LICENSE.md

@@ -0,0 +1,13 @@
+Copyright 2014 Sebastian YEPES FERNANDEZ
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License. You may
+obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing
+permissions and limitations under the License.

README.md

@@ -0,0 +1,66 @@
+CVM (Configuration Versioning Manager)
+================
+---
+CVM is an open source, generic device configuration collector that uses Git as its change historization engine.
+Its a more modern alternative to [RANCID](http://www.shrubbery.net/rancid/)
+
+A Brief Overview  of how CVM works:
+
+    - Retrieves the list of devices from either a static file (json) or dynamically from NNMi
+    - Logs into each device using the matching Authentication Profile taking advantage of Parallelism
+    - Runs the defined commands on the device and collects their expected output
+    - Cleans Up the output by removing sensitive and unnecessary data
+    - Individually saves the resulting output of each command locally in the file system
+    - A per device Git Repository is created and maintained with all the device commands
+
+
+## Notes
+    This is the first release of CVM and its currently working 100% for my use case. It still needs more work in the following areas:
+    - Rewrite the Device Profile assignation, right now profiles can only be assigned by vendor name but it should also be possible to assign them by device type or model (Something like the Device Authentication Profiles: deviceAuth.groovy)
+    - Support The Cisco privilege mode (enable mode) authentication
+    - Write more documentation on all the possible configurations options
+
+## Features
+- Collects and Processes device data concurrently (GPars)
+- Supports any SSH enabled device (v2+ Only)
+- Independent Git Repository for each collected device
+- Integration with [HP NNMi](http://www8.hp.com/us/en/software-solutions/network-node-manager-i-network-management-software/) for device discovery
+- Device Profile schema validator
+
+## Working Device Profiles
+- Cisco:
+  - Routers/Switchs/Nexus (Using TACACS+ enable mode account)
+- Fortinet
+  - FortiGate Firewalls
+- JuniperNetworks
+  - Juniper
+
+## Requirements
+- [Java](http://www.java.com) 1.7+
+- [Gradle](http://www.gradle.org) (Only if building the project from src)
+
+## Build/Run from master
+- clone repository
+- gradle (in project root)
+- The built release zip (build/dist/)
+
+## Installation and Configuration
+Take a look at the CVM [Wiki](https://github.com/syepes/cvm/wiki)
+
+## Contribute
+If you have any idea for an improvement or find a bug do not hesitate in opening an issue.
+And if you have the time clone this repo and submit a pull request to help improve the CVM project.
+
+## License
+CVM is distributed under Apache 2.0 License.
+
+## Used open source projects
+[Groovy](http://groovy.codehaus.org) |
+[GPars](http://gpars.codehaus.org) |
+[Logback](http://logback.qos.ch) |
+[JSch](http://www.jcraft.com/jsch) |
+[ExpectIt](https://github.com/Alexey1Gavrilov/ExpectIt) |
+[jGIT](http://www.eclipse.org/jgit) |
+[groovy-wslite](https://github.com/jwagenleitner/groovy-wslite) |
+[json-schema-validator](https://github.com/fge/json-schema-validator)
+

build.gradle

@@ -0,0 +1,143 @@
+buildscript {
+  repositories { jcenter() }
+  dependencies {
+    classpath 'org.kordamp.gradle:stats-gradle-plugin:0.1.+'
+  }
+}
+
+apply plugin:'groovy'
+apply plugin:'distribution'
+apply plugin:'org.kordamp.gradle.stats'
+
+def projectName = 'CVM'
+description = "$projectName - Configuration Versioning Manager"
+
+group = 'com.allthingsmonitoring.utils.CVM'
+archivesBaseName = 'cvm'
+distsDirName = 'dist'
+version = '1.0.0'
+
+def authorString = 'Sebastian YEPES FERNANDEZ ([email protected])'
+def copyrightString = 'Copyright © 2014 – Sebastian YEPES FERNANDEZ All Rights Reserved.'
+
+
+defaultTasks 'clean','build','groovydoc','distZip'
+
+
+// http://svn.codehaus.org/griffon/builders/jidebuilder/trunk/build.gradle
+// http://gparallelizer.googlecode.com/svn/trunk/build.gradle
+configure([compileGroovy, compileTestGroovy]) {
+  //groovyOptions.useAnt = true
+  //compileGroovy.groovyOptions.fork = true
+  groovyOptions.fork(memoryInitialSize: '128M', memoryMaximumSize: '1G')
+  groovyOptions.encoding = 'UTF-8'
+  //groovyOptions.optimizationOptions.indy = true
+}
+
+//configurations.compile.transitive = false
+compileJava {
+    options.encoding = 'UTF-8'
+    sourceCompatibility = 1.7
+    targetCompatibility = 1.7
+}
+
+
+repositories {
+  maven { url "http://oss.sonatype.org/content/repositories/snapshots/" }
+  maven { url 'http://oss.sonatype.org/content/groups/public/' }
+  maven { url 'http://repository.codehaus.org/' }
+  mavenLocal()
+  mavenCentral()
+}
+
+
+// 1.0.11 (GSiftingAppender.groovy) https://github.com/qos-ch/logback/commit/a86350e20ebf79b70aa737ea6149bf8485f2d295#logback-classic/src/main/groovy/ch/qos/logback
+dependencies {
+  compile 'org.codehaus.groovy:groovy:2.+',
+          'org.codehaus.groovy:groovy-jmx:2.+',
+          'org.codehaus.groovy:groovy-json:2.+',
+          'org.codehaus.groovy:groovy-xml:2.+',
+          'org.codehaus.groovy:groovy-nio:2.+',
+          'org.codehaus.groovy:groovy-ant:2.+',
+          'commons-cli:commons-cli:1.+',
+          'org.slf4j:slf4j-api:1.7.+',
+          'ch.qos.logback:logback-core:1.0.11',
+          'ch.qos.logback:logback-classic:1.0.11',
+          'com.jcraft:jsch:0.1.+',
+          'com.jcraft:jzlib:1.+',
+          'org.eclipse.jgit:org.eclipse.jgit:3.+',
+          'net.sf.expectit:expectit-core:0.+',
+          'com.github.fge:json-schema-validator:2.+',
+          fileTree(dir: 'libs', include: '*.jar')
+
+  compile 'com.github.groovy-wslite:groovy-wslite:1.+', {
+            exclude module: "groovy-all"
+          }
+  compile 'org.codehaus.jsr166-mirror:jsr166y:1.7.+'
+  compile 'org.codehaus.gpars:gpars:1.+', {
+            exclude module: "groovy-all"
+          }
+
+  testCompile "org.spockframework:spock-core:0.7-groovy-2.0-SNAPSHOT", {
+    exclude module: "groovy-all"
+  }
+}
+
+
+groovydoc {
+  dependsOn classes
+  groovyClasspath = configurations.compile // http://issues.gradle.org/browse/GRADLE-1391
+  destinationDir  = new File(( System.getProperty( 'groovydocDir' ) ?: project.buildDir.path + '/groovydoc' ), ( project.version.contains( '-SNAPSHOT' ) ? '' : project.version ))
+  use = true
+  includePrivate = true
+
+  windowTitle = "$projectName"
+  docTitle = "$projectName"
+  header = "$projectName - v$project.version"
+  footer = copyrightString
+
+  link 'http://static.springsource.org/spring/docs/3.1.x/javadoc-api/', 'org.springframework.'
+  link 'http://download.oracle.com/javase/6/docs/api/',                 'java.'
+  link 'http://groovy.codehaus.org/api/',                               'groovy.', 'org.codehaus.groovy.'
+  link 'http://commons.apache.org/net/apidocs/',                        'org.apache.commons.net.'
+  link 'http:/"truezip.java.net/apidocs/',                              'de.schlichtherle.'
+}
+
+
+processResources{
+  exclude 'config.groovy'
+  exclude '*.sh'
+}
+
+jar {
+  version = ''
+  from ( configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } ) {
+    exclude 'META-INF/*.SF'
+    exclude 'META-INF/*.DSA'
+    exclude 'META-INF/*.RSA'
+  }
+  manifest {
+    attributes 'Implementation-Title': project.description,
+               'Implementation-Version': project.version,
+               'Specification-Title': project.description,
+               'Specification-Version': project.version,
+               'Built-By': System.properties['user.name'],
+               'Built-JDK': System.properties['java.version'] +' ('+ System.properties['java.vm.vendor'] +')',
+               'Built-Date': new Date(),
+               'Author': authorString,
+               'Main-Class': project.group
+  }
+}
+
+distributions {
+    main {
+        baseName = archivesBaseName
+        contents {
+            from { 'build/libs' }
+            from { 'src/dist' }
+            from { 'README.md' }
+            from { 'CHANGELOG.md' }
+            from { 'LICENSE.md' }
+        }
+    }
+}

src/dist/authProfiles.groovy

@@ -0,0 +1,5 @@
+authProfiles = [[type:"vendor", pattern:"~/(?i)cisco.*/", auth:['cisco','secret']],
+                [type:"vendor", pattern:"~/(?i)fortinet.*/", auth:['admin','admin']],
+                [type:"vendor", pattern:"~/(?i)juniper.*/", auth:['netscreen','netscreen']],
+                [type:"device", pattern:"~/(?i)dc.-core.*/", auth:['admin','secret']]
+               ]

src/dist/cgit/ReadMe.md

@@ -0,0 +1 @@
+https://github.com/syepes/cvm/wiki/Installation-and-Configuration

src/dist/cgit/cgitrc

@@ -0,0 +1,37 @@
+# /etc/cgitrc
+
+virtual-root=/cvm
+root-title=CVM
+root-desc=Configuration Versioning Manager
+#root-readme=/files/web/example.com/code/about.html
+css=/cvm/cgit.css
+logo=/cvm/cvm.png
+#favicon=/cvm/favicon.ico
+# don't index site
+robots=noindex, nofollow
+
+# Cache
+cache-size=5000
+cache-root=/var/cache/cgit
+# No Cache
+#nocache=1
+#cache-size=0
+
+# Styling
+enable-index-links=1
+enable-index-owner=0
+enable-log-filecount=1
+enable-log-linecount=1
+enable-commit-graph=1
+enable-git-config=1
+
+max-stats=year
+max-commit-count=100
+max-message-length=100
+#side-by-side-diffs=1
+
+# Rpositories
+max-repo-count=100
+repository-sort=age
+snapshots=tar.gz zip
+scan-path=/opt/cvm/repository/

src/dist/cgit/cvm.conf

@@ -0,0 +1,27 @@
+Alias /cvm /var/www/htdocs/cgit/
+<Location /cvm>
+  AuthType Basic
+  AuthName "CVM - Configuration Versioning Manager"
+  AuthBasicProvider file
+  AuthUserFile /opt/cvm/.access
+  <LimitExcept OPTIONS>
+    Require valid-user
+  </LimitExcept>
+</Location>
+
+<Directory "/var/www/htdocs/cgit/">
+  AllowOverride None
+  Options ExecCGI FollowSymlinks
+  DirectoryIndex /cgi-bin/cgit.cgi
+  Order allow,deny
+  Allow from all
+
+  <IfModule mod_rewrite.c>
+    Options -MultiViews +SymLinksIfOwnerMatch
+    RewriteEngine On
+    RewriteCond %{REQUEST_FILENAME} !-f
+    RewriteCond %{REQUEST_FILENAME} !-d
+    RewriteRule ^(.*)$ /cgi-bin/cgit.cgi/$1 [L,PT]
+    RewriteRule ^$ /cgi-bin/cgit.cgi [L,PT]
+  </IfModule>
+</Directory>

src/dist/config.groovy

@@ -0,0 +1,17 @@
+// Absolute or Relative Paths
+git.repo                      = $/repository/$
+deviceProfilePath             = $/profiles/$
+authProfilePath               = $/authProfiles.groovy/$
+
+deviceSource.src              = 'NNMi' // file, NNMi
+
+// deviceSource.src = 'file'
+//deviceSource.file_path        = $//opt/cvm/deviceList.groovy/$
+
+// deviceSource.src = 'NNMi'
+deviceSource.nnmi_vip         = 'nnm.domain.com' // NNMi VIP
+deviceSource.nnmi_usr         = 'system'
+deviceSource.nnmi_pwd         = 'system'
+deviceSource.nnmi_deviceTypes = ['router','switchrouter','switch','firewall','wirelessaccesspoint']
+deviceSource.nnmi_nodeGroup   = 'Topology group' // NodeGroup or null
+

src/dist/cvm

@@ -0,0 +1,7 @@
+#!/bin/sh
+export JAVA_HOME=/usr/java/latest
+export PATH=$JAVA_HOME/bin:$PATH:.
+export JAVA_OPTS='-server -Xmx1G -Xms350M -XX:+AggressiveOpts -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:+UseCompressedOops'
+
+cd /opt/cvm/
+java -Dpid=$$ -Dapp.env=PROD -jar $PWD/cvm.jar $*

src/dist/cvm_cron

@@ -0,0 +1,8 @@
+#!/bin/sh
+#0 */12 * * * /opt/cvm/cvm_cron
+export JAVA_HOME=/usr/java/latest
+export PATH=$JAVA_HOME/bin:$PATH:.
+export JAVA_OPTS='-server -Xmx1G -Xms350M -XX:+AggressiveOpts -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:+UseCompressedOops'
+
+cd /opt/cvm/
+java -Dpid=$$ -Dapp.env=PROD -jar $PWD/cvm.jar >> $PWD/logs/cvm_schedule.log 2>&1

src/dist/deviceList.groovy

@@ -0,0 +1,7 @@
+deviceList = [
+              ['device':'ro1.domain.com', 'vendor':'Cisco', 'model':'1812', 'type':'Router'],
+              ['device':'ro2.domain.com', 'vendor':'Juniper', 'model':'XXX', 'type':'Router'],
+              ['device':'fw1.domain.com', 'vendor':'Fortigate', 'model':'50C', 'type':'Firewall'],
+              ['device':'fw2.domain.com', 'port':22, 'vendor':'Fortigate', 'model':'40C', 'type':'Firewall'],
+              ['device':'linux.domain.com', 'vendor':'Linux', 'model':'RedHat', 'type':'Server']
+             ]