From b7bb4dcf7d93e8a44587359abb5c219a4c294da1 Mon Sep 17 00:00:00 2001 From: Eric Torreborre Date: Tue, 7 Oct 2014 13:05:38 +1100 Subject: [PATCH] compiling for cdh3 --- project/dependencies.scala | 8 ++++---- .../com/nicta/scoobi/impl/io/Files.scala | 13 +++++-------- .../com/nicta/scoobi/io/FileSystemsSpec.scala | 19 ++++++++++--------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/project/dependencies.scala b/project/dependencies.scala index 1b25ce18d..964d759b9 100644 --- a/project/dependencies.scala +++ b/project/dependencies.scala @@ -42,10 +42,10 @@ object dependencies { def hadoop(version: String, hadoopVersion: String = "2.2.0") = - if (version.contains("cdh3")) Seq("com.nicta" %% "scoobi-compatibility-cdh3" % "1.0.2") - else if (version.contains("cdh4")) Seq("com.nicta" %% "scoobi-compatibility-cdh4" % "1.0.2") - else if (version.contains("cdh5")) Seq("com.nicta" %% "scoobi-compatibility-cdh5" % "1.0.2") - else Seq("com.nicta" %% "scoobi-compatibility-hadoop2" % "1.0.2") + if (version.contains("cdh3")) Seq("com.nicta" %% "scoobi-compatibility-cdh3" % "1.0.3") + else if (version.contains("cdh4")) Seq("com.nicta" %% "scoobi-compatibility-cdh4" % "1.0.3") + else if (version.contains("cdh5")) Seq("com.nicta" %% "scoobi-compatibility-cdh5" % "1.0.3") + else Seq("com.nicta" %% "scoobi-compatibility-hadoop2" % "1.0.3") def scalaz(scalazVersion: String = "7.0.6") = Seq( "org.scalaz" %% "scalaz-core" % scalazVersion, diff --git a/src/main/scala/com/nicta/scoobi/impl/io/Files.scala b/src/main/scala/com/nicta/scoobi/impl/io/Files.scala index 36fa3a263..770467a36 100644 --- a/src/main/scala/com/nicta/scoobi/impl/io/Files.scala +++ b/src/main/scala/com/nicta/scoobi/impl/io/Files.scala @@ -53,14 +53,14 @@ trait Files { val destPath = to.makeQualified(new Path(dirPath(dir.toString) + newPath)) if (!pathExists(destPath.getParent)) to.mkdirs(destPath.getParent) - if (List("s3n", "s3").contains(to.getScheme.toLowerCase)) + if (List("s3n", "s3").contains(Compatibility.getScheme(to).toLowerCase)) // s3 has special cases (can't rename, can't copy/rename dir simultaneously, ...) moveToS3(from, to, path, destPath) else if (sameFileSystem(from, to)) (path == destPath) || // same files - (from.isDirectory(path) && - to.isDirectory(destPath) && + (Compatibility.isDirectory(from, path) && + Compatibility.isDirectory(to, destPath) && path.toUri.getPath.startsWith(destPath.toUri.getPath)) || // nested directories { logger.debug(s"renaming $path to $destPath") @@ -114,12 +114,9 @@ trait Files { def fileStatus(path: Path)(implicit configuration: Configuration) = fileSystem(path).getFileStatus(path) /** @return true if the 2 fileSystems are the same */ - def sameFileSystem(from: FileSystem, to: FileSystem): Boolean = sameFileSystem(from.getUri, to.getUri) - - /** @return true if the 2 uri are one the same host with the same scheme */ - def sameFileSystem(from: URI, to: URI): Boolean = { + def sameFileSystem(from: FileSystem, to: FileSystem): Boolean = { def equalIgnoreCase(from: String, to: String) = (from == null && to == null) || from.equalsIgnoreCase(to) - equalIgnoreCase(from.getHost, to.getHost) && equalIgnoreCase(from.getScheme, to.getScheme) + equalIgnoreCase(from.getUri.getHost, to.getUri.getHost) && equalIgnoreCase(Compatibility.getScheme(from), Compatibility.getScheme(to)) } /** @return true if the file is a directory */ diff --git a/src/test/scala/com/nicta/scoobi/io/FileSystemsSpec.scala b/src/test/scala/com/nicta/scoobi/io/FileSystemsSpec.scala index 0695a54a4..1d2e72d60 100644 --- a/src/test/scala/com/nicta/scoobi/io/FileSystemsSpec.scala +++ b/src/test/scala/com/nicta/scoobi/io/FileSystemsSpec.scala @@ -18,7 +18,7 @@ package io import java.io.File import org.specs2.specification.Scope -import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.hadoop.fs.{FileSystem, FileStatus, Path} import impl.ScoobiConfiguration import com.nicta.scoobi.impl.io.{Files, FileSystems} import testing.mutable.UnitSpecification @@ -44,15 +44,16 @@ class FileSystemsSpec extends UnitSpecification with Tables { } "2 file systems are the same if they have the same host and same scheme" >> { val nullString: String = null - def uri(host: String, scheme: String) = new URI(scheme+"://"+host) + def uri(scheme: String, host: String) = + if (host == null) FileSystem.get(new URI(scheme+":"), new Configuration) + else FileSystem.get(new URI(scheme+"://"+host+":3100/"), new Configuration) - "host1" | "scheme1" | "host2" | "scheme2" | "same?" |> - "local" ! "file" ! "local" ! "file" ! true | - "local" ! "hdfs" ! "local" ! "file" ! false | - "local" ! "file" ! "cluster" ! "file" ! false | - nullString ! nullString ! nullString ! nullString ! true | - nullString ! "file" ! "local" ! "file" ! false | { (h1, s1, h2, s2, same) => - Files.sameFileSystem(uri(h1, s1), uri(h2, s2)) === same + "scheme1" | "host1" | "scheme1" | "host2" | "same?" |> + "file" ! "localhost" ! "file" ! "local" ! true | + "hdfs" ! "localhost" ! "file" ! "local" ! false | + "hdfs" ! "localhost" ! "hdfs" ! "google.com" ! false | + "hdfs" ! "localhost" ! "file" ! "google.com" ! false | { (s1, h1, s2, h2, same) => + Files.sameFileSystem(uri(s1, h1), uri(s2, h2)) === same } }