compiling for cdh3

NICTA · Oct 7, 2014 · b7bb4dc · b7bb4dc
1 parent b36ae07
commit b7bb4dc
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 21 deletions.
diff --git a/project/dependencies.scala b/project/dependencies.scala
@@ -42,10 +42,10 @@ object dependencies {
 
   def hadoop(version: String, hadoopVersion: String = "2.2.0") =
 
-    if (version.contains("cdh3"))      Seq("com.nicta" %% "scoobi-compatibility-cdh3"    % "1.0.2")
-    else if (version.contains("cdh4")) Seq("com.nicta" %% "scoobi-compatibility-cdh4"    % "1.0.2")
-    else if (version.contains("cdh5")) Seq("com.nicta" %% "scoobi-compatibility-cdh5"    % "1.0.2")
-    else                               Seq("com.nicta" %% "scoobi-compatibility-hadoop2" % "1.0.2")
+    if (version.contains("cdh3"))      Seq("com.nicta" %% "scoobi-compatibility-cdh3"    % "1.0.3")
+    else if (version.contains("cdh4")) Seq("com.nicta" %% "scoobi-compatibility-cdh4"    % "1.0.3")
+    else if (version.contains("cdh5")) Seq("com.nicta" %% "scoobi-compatibility-cdh5"    % "1.0.3")
+    else                               Seq("com.nicta" %% "scoobi-compatibility-hadoop2" % "1.0.3")
 
   def scalaz(scalazVersion: String = "7.0.6") = Seq(
     "org.scalaz"                        %% "scalaz-core"               % scalazVersion,

diff --git a/src/main/scala/com/nicta/scoobi/impl/io/Files.scala b/src/main/scala/com/nicta/scoobi/impl/io/Files.scala
@@ -53,14 +53,14 @@ trait Files {
       val destPath = to.makeQualified(new Path(dirPath(dir.toString) + newPath))
       if (!pathExists(destPath.getParent)) to.mkdirs(destPath.getParent)
 
-      if (List("s3n", "s3").contains(to.getScheme.toLowerCase))
+      if (List("s3n", "s3").contains(Compatibility.getScheme(to).toLowerCase))
         // s3 has special cases (can't rename, can't copy/rename dir simultaneously, ...)
         moveToS3(from, to, path, destPath)
 
       else if (sameFileSystem(from, to))
         (path == destPath) || // same files
-        (from.isDirectory(path) &&
-          to.isDirectory(destPath) &&
+        (Compatibility.isDirectory(from, path) &&
+         Compatibility.isDirectory(to, destPath) &&
           path.toUri.getPath.startsWith(destPath.toUri.getPath)) || // nested directories
         {
           logger.debug(s"renaming $path to $destPath")
@@ -114,12 +114,9 @@ trait Files {
   def fileStatus(path: Path)(implicit configuration: Configuration) = fileSystem(path).getFileStatus(path)
 
   /** @return true if the 2 fileSystems are the same */
-  def sameFileSystem(from: FileSystem, to: FileSystem): Boolean = sameFileSystem(from.getUri, to.getUri)
-
-  /** @return true if the 2 uri are one the same host with the same scheme */
-  def sameFileSystem(from: URI, to: URI): Boolean = {
+  def sameFileSystem(from: FileSystem, to: FileSystem): Boolean = {
     def equalIgnoreCase(from: String, to: String) = (from == null && to == null) || from.equalsIgnoreCase(to)
-    equalIgnoreCase(from.getHost, to.getHost) && equalIgnoreCase(from.getScheme, to.getScheme)
+    equalIgnoreCase(from.getUri.getHost, to.getUri.getHost) && equalIgnoreCase(Compatibility.getScheme(from), Compatibility.getScheme(to))
   }
 
   /** @return true if the file is a directory */

diff --git a/src/test/scala/com/nicta/scoobi/io/FileSystemsSpec.scala b/src/test/scala/com/nicta/scoobi/io/FileSystemsSpec.scala
@@ -18,7 +18,7 @@ package io
 
 import java.io.File
 import org.specs2.specification.Scope
-import org.apache.hadoop.fs.{FileStatus, Path}
+import org.apache.hadoop.fs.{FileSystem, FileStatus, Path}
 import impl.ScoobiConfiguration
 import com.nicta.scoobi.impl.io.{Files, FileSystems}
 import testing.mutable.UnitSpecification
@@ -44,15 +44,16 @@ class FileSystemsSpec extends UnitSpecification with Tables {
   }
   "2 file systems are the same if they have the same host and same scheme" >> {
     val nullString: String = null
-    def uri(host: String, scheme: String) = new URI(scheme+"://"+host)
+    def uri(scheme: String, host: String) =
+      if (host == null) FileSystem.get(new URI(scheme+":"), new Configuration)
+      else              FileSystem.get(new URI(scheme+"://"+host+":3100/"), new Configuration)
 
-    "host1"    | "scheme1"  | "host2"    | "scheme2"  | "same?" |>
-    "local"    ! "file"     ! "local"    ! "file"     ! true    |
-    "local"    ! "hdfs"     ! "local"    ! "file"     ! false   |
-    "local"    ! "file"     ! "cluster"  ! "file"     ! false   |
-    nullString ! nullString ! nullString ! nullString ! true    |
-    nullString ! "file"     ! "local"    ! "file"     ! false   | { (h1, s1, h2, s2, same) =>
-      Files.sameFileSystem(uri(h1, s1), uri(h2, s2)) === same
+    "scheme1" | "host1"     | "scheme1" | "host2"      | "same?" |>
+    "file"    ! "localhost" ! "file"    ! "local"      ! true    |
+    "hdfs"    ! "localhost" ! "file"    ! "local"      ! false   |
+    "hdfs"    ! "localhost" ! "hdfs"    ! "google.com" ! false   |
+    "hdfs"    ! "localhost" ! "file"    ! "google.com" ! false   | { (s1, h1, s2, h2, same) =>
+      Files.sameFileSystem(uri(s1, h1), uri(s2, h2)) === same
     }
   }