diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f4e6d93..eea9e946 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ -1.5.5 - xxxxxxxxxxxxxxxxx -========================= +1.5.5 - March 4, 2015 +===================== * Issue 231: Possible NPE due to not checking for null from usState.getUs() diff --git a/README.md b/README.md index 6b11766d..6922b8cc 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ There is an Exhibitor mailing list. Join here: http://groups.google.com/group/ex ## AUTHOR -Jordan Zimmerman (jzimmerman@netflix.com) +Jordan Zimmerman (jordan@jordanzimmerman.com) ## LICENSE diff --git a/exhibitor-core/build.gradle b/exhibitor-core/build.gradle index a1470e6b..50f50252 100644 --- a/exhibitor-core/build.gradle +++ b/exhibitor-core/build.gradle @@ -23,6 +23,7 @@ dependencies { compile 'org.codehaus.jsr166-mirror:jsr166y:1.7.0' compile "com.amazonaws:aws-java-sdk:${awsVersion}" + compile "com.microsoft.azure:azure-storage:${azureVersion}" compile "com.sun.jersey:jersey-core:${jerseyVersion}" compile "com.sun.jersey:jersey-server:${jerseyVersion}" compile "com.sun.jersey:jersey-json:${jerseyVersion}" diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureClient.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureClient.java new file mode 100644 index 00000000..fca6942b --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureClient.java @@ -0,0 +1,23 @@ +package com.netflix.exhibitor.core.azure; + +import com.microsoft.azure.storage.blob.BlobProperties; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.blob.ListBlobItem; + +import java.io.Closeable; + +public interface AzureClient extends Closeable { + public CloudBlobClient getClient() throws Exception; + + public CloudBlob getBlob(String containerName, String uri) throws Exception; + + public BlobProperties getBlobProperties(String containerName, String uri) throws Exception; + + public Iterable listBlobs(String containerName, String prefix) throws Exception; + + public void putBlob(byte[] bytes, String containerName, String uri) throws Exception; + + public void deleteBlob(String containerName, String uri) throws Exception; + +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureClientFactory.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureClientFactory.java new file mode 100644 index 00000000..198cc594 --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureClientFactory.java @@ -0,0 +1,13 @@ +package com.netflix.exhibitor.core.azure; + +public interface AzureClientFactory { + /** + * Create a client with the given credentials + * + * @param credentials credentials + * @return client + * @throws Exception errors + */ + public AzureClient makeNewClient(AzureCredential credentials) throws Exception; + +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureClientFactoryImpl.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureClientFactoryImpl.java new file mode 100644 index 00000000..e8e5139e --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureClientFactoryImpl.java @@ -0,0 +1,9 @@ +package com.netflix.exhibitor.core.azure; + + +public class AzureClientFactoryImpl implements AzureClientFactory { + @Override + public AzureClient makeNewClient(AzureCredential credentials) throws Exception { + return new AzureClientImpl(credentials); + } +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureClientImpl.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureClientImpl.java new file mode 100644 index 00000000..7c89b802 --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureClientImpl.java @@ -0,0 +1,63 @@ +package com.netflix.exhibitor.core.azure; + +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.blob.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +public class AzureClientImpl implements AzureClient { + private final Logger log = LoggerFactory.getLogger(getClass()); + private final String storageConnectionString; + + public AzureClientImpl(AzureCredential credentials) { + this.storageConnectionString = + "DefaultEndpointsProtocol=http" + + ";AccountName=" + credentials.getAccountName() + + ";AccountKey=" + credentials.getAccountKey(); + } + + @Override + public CloudBlobClient getClient() throws Exception { + CloudStorageAccount storageAccount = CloudStorageAccount.parse(this.storageConnectionString); + return storageAccount.createCloudBlobClient(); + } + + @Override + public CloudBlob getBlob(String containerName, String uri) throws Exception { + CloudBlobContainer container = getClient().getContainerReference(containerName); + return container.getBlockBlobReference(uri); + } + + @Override + public BlobProperties getBlobProperties(String containerName, String uri) throws Exception { + CloudBlob blob = getBlob(containerName, uri); + blob.downloadAttributes(); + return blob.getProperties(); + } + + @Override + public Iterable listBlobs(String containerName, String prefix) throws Exception { + CloudBlobContainer container = getClient().getContainerReference(containerName); + return container.listBlobs(prefix); + } + + @Override + public void putBlob(byte[] bytes, String containerName, String uri) throws Exception { + CloudBlob blob = getBlob(containerName, uri); + blob.getContainer().createIfNotExists(); + blob.upload(new ByteArrayInputStream(bytes), bytes.length); + } + + @Override + public void deleteBlob(String containerName, String uri) throws Exception { + getBlob(containerName, uri).deleteIfExists(); + } + + @Override + public void close() throws IOException { + + } +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureCredential.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureCredential.java new file mode 100644 index 00000000..3a1927e6 --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/AzureCredential.java @@ -0,0 +1,6 @@ +package com.netflix.exhibitor.core.azure; + +public interface AzureCredential { + public String getAccountName(); + public String getAccountKey(); +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/PropertyBasedAzureCredential.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/PropertyBasedAzureCredential.java new file mode 100644 index 00000000..b29c36e6 --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/azure/PropertyBasedAzureCredential.java @@ -0,0 +1,51 @@ +package com.netflix.exhibitor.core.azure; + +import org.apache.curator.utils.CloseableUtils; + +import java.io.*; +import java.util.Properties; + +public class PropertyBasedAzureCredential implements AzureCredential { + private final String accountName; + private final String accountKey; + + public static final String PROPERTY_AZURE_ACCOUNT_NAME = "com.netflix.exhibitor.azure.account-name"; + public static final String PROPERTY_AZURE_ACCOUNT_KEY = "com.netflix.exhibitor.azure.account-key"; + + public PropertyBasedAzureCredential(File propertiesFile) throws IOException + { + this(loadProperties(propertiesFile)); + } + + public PropertyBasedAzureCredential(Properties properties) + { + accountName = properties.getProperty(PROPERTY_AZURE_ACCOUNT_NAME); + accountKey = properties.getProperty(PROPERTY_AZURE_ACCOUNT_KEY); + } + + @Override + public String getAccountName() + { + return accountName; + } + + public String getAccountKey() + { + return accountKey; + } + + private static Properties loadProperties(File propertiesFile) throws IOException + { + Properties properties = new Properties(); + InputStream in = new BufferedInputStream(new FileInputStream(propertiesFile)); + try + { + properties.load(in); + } + finally + { + CloseableUtils.closeQuietly(in); + } + return properties; + } +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/PseudoLockBase.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/PseudoLockBase.java index 81f98efa..4f58b8b1 100644 --- a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/PseudoLockBase.java +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/PseudoLockBase.java @@ -55,7 +55,7 @@ public abstract class PseudoLockBase implements PseudoLock /** * @param lockPrefix key prefix * @param timeoutMs max age for locks - * @param pollingMs how often to poll S3 + * @param pollingMs how often to poll storage service */ public PseudoLockBase(String lockPrefix, int timeoutMs, int pollingMs) { @@ -65,8 +65,8 @@ public PseudoLockBase(String lockPrefix, int timeoutMs, int pollingMs) /** * @param lockPrefix key prefix * @param timeoutMs max age for locks - * @param pollingMs how often to poll S3 - * @param settlingMs how long to wait for S3 to reach consistency + * @param pollingMs how often to poll storage service + * @param settlingMs how long to wait for storage service to reach consistency */ public PseudoLockBase(String lockPrefix, int timeoutMs, int pollingMs, int settlingMs) { diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/azure/AzureConfigArguments.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/azure/AzureConfigArguments.java new file mode 100644 index 00000000..86a3cf56 --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/azure/AzureConfigArguments.java @@ -0,0 +1,29 @@ +package com.netflix.exhibitor.core.config.azure; + +public class AzureConfigArguments { + private final String container; + private final String blobName; + private final AzureConfigAutoManageLockArguments lockArguments; + + public AzureConfigArguments(String container, String blobName, AzureConfigAutoManageLockArguments lockArguments) + { + this.container = container; + this.blobName = blobName; + this.lockArguments = lockArguments; + } + + public String getContainer() + { + return container; + } + + public String getBlobName() + { + return blobName; + } + + public AzureConfigAutoManageLockArguments getLockArguments() + { + return lockArguments; + } +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/azure/AzureConfigAutoManageLockArguments.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/azure/AzureConfigAutoManageLockArguments.java new file mode 100644 index 00000000..bb8e0d56 --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/azure/AzureConfigAutoManageLockArguments.java @@ -0,0 +1,26 @@ +package com.netflix.exhibitor.core.config.azure; + +import com.netflix.exhibitor.core.config.AutoManageLockArguments; +import java.util.concurrent.TimeUnit; + +public class AzureConfigAutoManageLockArguments extends AutoManageLockArguments +{ + private final int settlingMs; + + public AzureConfigAutoManageLockArguments(String prefix) + { + super(prefix); + settlingMs = (int)TimeUnit.SECONDS.toMillis(5); // TODO - get this right + } + + public AzureConfigAutoManageLockArguments(String prefix, int timeoutMs, int pollingMs, int settlingMs) + { + super(prefix, timeoutMs, pollingMs); + this.settlingMs = settlingMs; + } + + public int getSettlingMs() + { + return settlingMs; + } +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/azure/AzureConfigProvider.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/azure/AzureConfigProvider.java new file mode 100644 index 00000000..5cdabe48 --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/azure/AzureConfigProvider.java @@ -0,0 +1,158 @@ +package com.netflix.exhibitor.core.config.azure; + +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlobProperties; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.netflix.exhibitor.core.azure.*; +import com.netflix.exhibitor.core.config.*; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Date; +import java.util.Properties; + +public class AzureConfigProvider implements ConfigProvider { + private final AzureConfigArguments arguments; + private final AzureClient azureClient; + private final String hostname; + private final Properties defaults; + + public static final String AUTO_GENERATED_NOTE = "Auto-generated by Exhibitor on %s"; + public static final int HTTP_FORBIDDEN = 403; + public static final int HTTP_NOT_FOUND = 404; + + /** + * @param factory the factory + * @param credentials credentials + * @param arguments args + * @param hostname this VM's hostname + * @throws Exception errors + */ + public AzureConfigProvider(AzureClientFactory factory, AzureCredential credentials, AzureConfigArguments arguments, + String hostname) throws Exception { + this(factory, credentials, arguments, hostname, new Properties()); + } + + /** + * @param factory the factory + * @param credentials credentials + * @param arguments args + * @param hostname this VM's hostname + * @param defaults default props + * @throws Exception errors + */ + public AzureConfigProvider(AzureClientFactory factory, AzureCredential credentials, AzureConfigArguments arguments, + String hostname, Properties defaults) throws Exception { + this.arguments = arguments; + this.hostname = hostname; + this.defaults = defaults; + azureClient = factory.makeNewClient(credentials); + } + + public AzureClient getAzureClient() { + return azureClient; + } + + @Override + public void start() throws Exception { + // NOP + } + + @Override + public void close() throws IOException { + azureClient.close(); + } + + @Override + public PseudoLock newPseudoLock() throws Exception { + return new AzurePseudoLock + ( + azureClient, + arguments.getContainer(), + arguments.getLockArguments().getPrefix(), + arguments.getLockArguments().getTimeoutMs(), + arguments.getLockArguments().getPollingMs(), + arguments.getLockArguments().getSettlingMs() + ); + } + + @Override + public LoadedInstanceConfig loadConfig() throws Exception { + Date lastModified; + Properties properties = new Properties(); + CloudBlob configObject = getConfigObject(); + if (configObject != null) { + lastModified = configObject.getProperties().getLastModified(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + configObject.download(out); + properties.load(new ByteArrayInputStream(out.toByteArray())); + } else { + lastModified = new Date(0L); + } + + PropertyBasedInstanceConfig config = new PropertyBasedInstanceConfig(properties, defaults); + return new LoadedInstanceConfig(config, lastModified.getTime()); + } + + @Override + public LoadedInstanceConfig storeConfig(ConfigCollection config, long compareVersion) throws Exception { + { + BlobProperties blobProperties = getConfigBlobProperties(); + if (blobProperties != null) { + Date lastModified = blobProperties.getLastModified(); + if (lastModified.getTime() != compareVersion) { + return null; // pattern copied from S3ConfigProvider - Azure may support a better way + } + } + } + + PropertyBasedInstanceConfig propertyBasedInstanceConfig = new PropertyBasedInstanceConfig(config); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + propertyBasedInstanceConfig.getProperties().store(out, String.format(AUTO_GENERATED_NOTE, hostname)); + + byte[] bytes = out.toByteArray(); + azureClient.putBlob(bytes, arguments.getContainer(), arguments.getBlobName()); + + BlobProperties blobProperties = azureClient.getBlobProperties(arguments.getContainer(), arguments.getBlobName()); + return new LoadedInstanceConfig(propertyBasedInstanceConfig, blobProperties.getLastModified().getTime()); + } + + private BlobProperties getConfigBlobProperties() throws Exception { + try { + BlobProperties properties = azureClient.getBlobProperties(arguments.getContainer(), arguments.getBlobName()); + if (properties.getLength() > 0) { + return properties; + } + } catch (StorageException e) { + if (!isNotFoundError(e) && !isForbiddenError(e)) { + throw e; + } + } + return null; + } + + private CloudBlob getConfigObject() throws Exception { + try { + CloudBlob object = azureClient.getBlob(arguments.getContainer(), arguments.getBlobName()); + object.downloadAttributes(); + if (object.getProperties().getLength() > 0) { + return object; + } + } catch (StorageException e) { + if (!isNotFoundError(e) && !isForbiddenError(e)) { + throw e; + } + } + return null; + } + + private boolean isForbiddenError(StorageException e) { + return ((e.getHttpStatusCode() == HTTP_FORBIDDEN)); + } + + private boolean isNotFoundError(StorageException e) { + return (e.getHttpStatusCode() == HTTP_NOT_FOUND); + } + +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/azure/AzurePseudoLock.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/azure/AzurePseudoLock.java new file mode 100644 index 00000000..e733b162 --- /dev/null +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/azure/AzurePseudoLock.java @@ -0,0 +1,62 @@ +package com.netflix.exhibitor.core.config.azure; + +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.ListBlobItem; +import com.netflix.exhibitor.core.azure.AzureClient; +import com.netflix.exhibitor.core.config.PseudoLockBase; + +import java.util.ArrayList; +import java.util.List; + +public class AzurePseudoLock extends PseudoLockBase +{ + private final AzureClient client; + private final String container; + + /** + * @param client the Azure client + * @param container the Azure container + * @param lockPrefix the Azure blob prefix + * @param timeoutMs max age for locks + * @param pollingMs how often to poll Azure + * @param settlingMs how long to wait for Azure to reach consistency + */ + public AzurePseudoLock(AzureClient client, String container, String lockPrefix, int timeoutMs, int pollingMs, int settlingMs) + { + super(lockPrefix, timeoutMs, pollingMs, settlingMs); + this.client = client; + this.container = container; + } + + @Override + protected void createFile(String uri, byte[] contents) throws Exception + { + client.putBlob(contents, container, uri); + } + + @Override + protected void deleteFile(String uri) throws Exception + { + try + { + client.deleteBlob(container, uri); + } + catch ( StorageException ignore ) + { + // ignore these + } + } + + @Override + protected List getFileNames(String lockPrefix) throws Exception { + Iterable blobs = client.listBlobs(container, lockPrefix); + + List fileNames = new ArrayList(); + for (ListBlobItem blob : blobs) { + fileNames.add(((CloudBlockBlob) blob).getName()); + } + + return fileNames; + } +} diff --git a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/s3/S3ConfigProvider.java b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/s3/S3ConfigProvider.java index b9c01a9b..8b4cd4a2 100644 --- a/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/s3/S3ConfigProvider.java +++ b/exhibitor-core/src/main/java/com/netflix/exhibitor/core/config/s3/S3ConfigProvider.java @@ -127,6 +127,7 @@ public S3ConfigProvider(S3ClientFactory factory, S3Credential credential, S3Clie * @param clientConfig s3 client configuration * @param arguments args * @param hostname this VM's hostname + * @param defaults default props * @param s3Region optional region or null * @throws Exception errors */ diff --git a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java index 3ee5febb..c0e7acf1 100644 --- a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java +++ b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCLI.java @@ -21,6 +21,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.netflix.exhibitor.core.Exhibitor; +import com.netflix.exhibitor.core.azure.PropertyBasedAzureCredential; import com.netflix.exhibitor.core.config.IntConfigs; import com.netflix.exhibitor.core.config.JQueryStyle; import com.netflix.exhibitor.core.config.PropertyBasedInstanceConfig; @@ -71,6 +72,9 @@ private OptionSection(String sectionName, Options options) public static final String S3_CONFIG = "s3config"; public static final String S3_CONFIG_PREFIX = "s3configprefix"; public static final String S3_REGION = "s3region"; + public static final String AZURE_CONFIG_PREFIX = "azureconfigprefix"; + public static final String AZURE_CREDENTIALS = "azurecredentials"; + public static final String AZURE_CONFIG = "azureconfig"; public static final String ZOOKEEPER_CONFIG_INITIAL_CONNECT_STRING = "zkconfigconnect"; public static final String ZOOKEEPER_CONFIG_EXHIBITOR_PORT = "zkconfigexhibitorport"; public static final String ZOOKEEPER_CONFIG_EXHIBITOR_URI_PATH = "zkconfigexhibitorpath"; @@ -138,6 +142,9 @@ public ExhibitorCLI() s3ConfigOptions.addOption(null, S3_CONFIG, true, "The bucket name and key to store the config (s3credentials may be provided as well). Argument is [bucket name]:[key]."); s3ConfigOptions.addOption(null, S3_CONFIG_PREFIX, true, "When using AWS S3 shared config files, the prefix to use for values such as locks. Default is " + DEFAULT_PREFIX); + Options azureConfigOptions = new Options(); + azureConfigOptions.addOption(null, AZURE_CONFIG, true, "The container name and blob name to store the config (azurecredentials may be provided as well). Argument is [container name]:[blob name]."); + Options zookeeperConfigOptions = new Options(); zookeeperConfigOptions.addOption(null, ZOOKEEPER_CONFIG_INITIAL_CONNECT_STRING, true, "The initial connection string for ZooKeeper shared config storage. E.g: \"host1:2181,host2:2181...\""); zookeeperConfigOptions.addOption(null, ZOOKEEPER_CONFIG_EXHIBITOR_PORT, true, "Used if the ZooKeeper shared config is also running Exhibitor. This is the port that Exhibitor is listening on. IMPORTANT: if this value is not set it implies that Exhibitor is not being used on the ZooKeeper shared config."); @@ -158,6 +165,9 @@ public ExhibitorCLI() s3Options.addOption(null, S3_REGION, true, "Optional region for S3 calls (e.g. \"eu-west-1\"). Will be used to set the S3 client's endpoint."); s3Options.addOption(null, S3_PROXY, true, "Optional configuration used when when connecting to S3 via a proxy. Argument is the path to an AWS credential properties file with four properties (only host, port and protocol are required if using a proxy): " + PropertyBasedS3ClientConfig.PROPERTY_S3_PROXY_HOST + ", " + PropertyBasedS3ClientConfig.PROPERTY_S3_PROXY_PORT + ", " + PropertyBasedS3ClientConfig.PROPERTY_S3_PROXY_USERNAME + ", " + PropertyBasedS3ClientConfig.PROPERTY_S3_PROXY_PASSWORD); + Options azureOptions = new Options(); + azureOptions.addOption(null, AZURE_CREDENTIALS, true, "Optional credentials to use for azureconfig. Argument is the path to an Azure credential properties file with two properties: " + PropertyBasedAzureCredential.PROPERTY_AZURE_ACCOUNT_NAME + " and " + PropertyBasedAzureCredential.PROPERTY_AZURE_ACCOUNT_KEY); + generalOptions = new Options(); generalOptions.addOption(null, TIMEOUT, true, "Connection timeout (ms) for ZK connections. Default is 30000."); generalOptions.addOption(null, LOGLINES, true, "Max lines of logging to keep in memory for display. Default is 1000."); @@ -180,9 +190,11 @@ public ExhibitorCLI() options = new Options(); addAll("S3 Options", s3Options); + addAll("Azure Options", azureOptions); addAll("Configuration Options for Type \"s3\"", s3ConfigOptions); addAll("Configuration Options for Type \"zookeeper\"", zookeeperConfigOptions); addAll("Configuration Options for Type \"file\"", fileConfigOptions); + addAll("Configuration Options for Type \"azure\"", azureConfigOptions); addAll("Configuration Options for Type \"none\"", noneConfigOptions); addAll("Backup Options", backupOptions); addAll("Authorization Options", authOptions); diff --git a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java index da90e1da..b3d57ad3 100644 --- a/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java +++ b/exhibitor-standalone/src/main/java/com/netflix/exhibitor/standalone/ExhibitorCreator.java @@ -20,6 +20,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.netflix.exhibitor.core.ExhibitorArguments; +import com.netflix.exhibitor.core.azure.AzureClientFactoryImpl; +import com.netflix.exhibitor.core.azure.PropertyBasedAzureCredential; import com.netflix.exhibitor.core.backup.BackupProvider; import com.netflix.exhibitor.core.backup.filesystem.FileSystemBackupProvider; import com.netflix.exhibitor.core.backup.s3.S3BackupProvider; @@ -30,6 +32,9 @@ import com.netflix.exhibitor.core.config.JQueryStyle; import com.netflix.exhibitor.core.config.PropertyBasedInstanceConfig; import com.netflix.exhibitor.core.config.StringConfigs; +import com.netflix.exhibitor.core.config.azure.AzureConfigArguments; +import com.netflix.exhibitor.core.config.azure.AzureConfigAutoManageLockArguments; +import com.netflix.exhibitor.core.config.azure.AzureConfigProvider; import com.netflix.exhibitor.core.config.filesystem.FileSystemConfigProvider; import com.netflix.exhibitor.core.config.none.NoneConfigProvider; import com.netflix.exhibitor.core.config.s3.S3ConfigArguments; @@ -131,6 +136,13 @@ public ExhibitorCreator(String[] args) throws Exception awsClientConfig = new PropertyBasedS3ClientConfig(new File(commandLine.getOptionValue(S3_PROXY))); } + PropertyBasedAzureCredential azureCredentials = null; + if ( commandLine.hasOption(AZURE_CREDENTIALS) ) + { + azureCredentials = new PropertyBasedAzureCredential(new File(commandLine.getOptionValue(AZURE_CREDENTIALS))); + } + + BackupProvider backupProvider = null; if ( "true".equalsIgnoreCase(commandLine.getOptionValue(S3_BACKUP)) ) { @@ -155,7 +167,7 @@ else if ( "true".equalsIgnoreCase(commandLine.getOptionValue(FILESYSTEMBACKUP)) throw new MissingConfigurationTypeException("Configuration type (-" + SHORT_CONFIG_TYPE + " or --" + CONFIG_TYPE + ") must be specified", cli); } - ConfigProvider configProvider = makeConfigProvider(configType, cli, commandLine, awsCredentials, awsClientConfig, backupProvider, useHostname, s3Region); + ConfigProvider configProvider = makeConfigProvider(configType, cli, commandLine, awsCredentials, awsClientConfig, backupProvider, useHostname, s3Region, azureCredentials); if ( configProvider == null ) { throw new ExhibitorCreatorExit(cli); @@ -278,7 +290,7 @@ public String getRemoteAuthSpec() return remoteAuthSpec; } - private ConfigProvider makeConfigProvider(String configType, ExhibitorCLI cli, CommandLine commandLine, PropertyBasedS3Credential awsCredentials, PropertyBasedS3ClientConfig awsClientConfig, BackupProvider backupProvider, String useHostname, String s3Region) throws Exception + private ConfigProvider makeConfigProvider(String configType, ExhibitorCLI cli, CommandLine commandLine, PropertyBasedS3Credential awsCredentials, PropertyBasedS3ClientConfig awsClientConfig, BackupProvider backupProvider, String useHostname, String s3Region, PropertyBasedAzureCredential azureCredentials) throws Exception { Properties defaultProperties = makeDefaultProperties(commandLine, backupProvider); @@ -295,6 +307,10 @@ else if ( configType.equals("zookeeper") ) { configProvider = getZookeeperProvider(commandLine, useHostname, defaultProperties); } + else if ( configType.equals("azure") ) + { + configProvider = getAzureProvider(cli, commandLine, azureCredentials, useHostname, defaultProperties); + } else if ( configType.equals("none") ) { log.warn("Warning: you have intentionally turned off shared configuration. This mode is meant for special purposes only. Please verify that this is your intent."); @@ -528,6 +544,12 @@ private ConfigProvider getS3Provider(ExhibitorCLI cli, CommandLine commandLine, return new S3ConfigProvider(new S3ClientFactoryImpl(), awsCredentials, awsClientConfig, getS3Arguments(cli, commandLine.getOptionValue(S3_CONFIG), prefix), hostname, defaultProperties, s3Region); } + private ConfigProvider getAzureProvider(ExhibitorCLI cli, CommandLine commandLine, PropertyBasedAzureCredential azureCredentials, String hostname, Properties defaultProperties) throws Exception + { + String prefix = cli.getOptions().hasOption(AZURE_CONFIG_PREFIX) ? commandLine.getOptionValue(AZURE_CONFIG_PREFIX) : DEFAULT_PREFIX; + return new AzureConfigProvider(new AzureClientFactoryImpl(), azureCredentials, getAzureArguments(cli, commandLine.getOptionValue(AZURE_CONFIG), prefix), hostname, defaultProperties); + } + private void checkMutuallyExclusive(ExhibitorCLI cli, CommandLine commandLine, String option1, String option2) throws ExhibitorCreatorExit { if ( commandLine.hasOption(option1) && commandLine.hasOption(option2) ) @@ -548,6 +570,17 @@ private S3ConfigArguments getS3Arguments(ExhibitorCLI cli, String value, String return new S3ConfigArguments(parts[0].trim(), parts[1].trim(), new S3ConfigAutoManageLockArguments(prefix + "-lock-")); } + private AzureConfigArguments getAzureArguments(ExhibitorCLI cli, String value, String prefix) throws ExhibitorCreatorExit + { + String[] parts = value.split(":"); + if ( parts.length != 2 ) + { + log.error("Bad azureconfig argument: " + value); + throw new ExhibitorCreatorExit(cli); + } + return new AzureConfigArguments(parts[0].trim(), parts[1].trim(), new AzureConfigAutoManageLockArguments(prefix + "-lock-")); + } + private CuratorFramework makeCurator(final String connectString, int baseSleepTimeMs, int maxRetries, int exhibitorPort, String exhibitorRestPath, int pollingMs) { List hostnames = Lists.newArrayList(); diff --git a/exhibitor-standalone/src/main/resources/buildscripts/standalone/maven/pom.xml b/exhibitor-standalone/src/main/resources/buildscripts/standalone/maven/pom.xml index 295767b0..08354711 100644 --- a/exhibitor-standalone/src/main/resources/buildscripts/standalone/maven/pom.xml +++ b/exhibitor-standalone/src/main/resources/buildscripts/standalone/maven/pom.xml @@ -3,13 +3,13 @@ 4.0.0 exhibitor exhibitor - 1.0 + 1.5.5 com.netflix.exhibitor exhibitor-standalone - 1.5.4 + ${project.version} @@ -33,7 +33,10 @@ - com.netflix.exhibitor.application.ExhibitorMain + + com.netflix.exhibitor.application.ExhibitorMain + ${project.version} + diff --git a/gradle.properties b/gradle.properties index 06085ea0..6e59b26f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,4 +11,5 @@ jaxRsVersion=1.1.1 jacksonVersion=1.8.3 luceneVersion=3.6.0 awsVersion=1.9.3 +azureVersion=2.0.0 mockitoVersion=1.8.5