From bbb5f2d6a1d76feec10e659ba7065a859baf4db2 Mon Sep 17 00:00:00 2001
From: pascal Christoph <christoph@hbz-nrw.de>
Date: Thu, 30 Jan 2014 14:56:25 +0100
Subject: [PATCH 1/2] Add catching exception and add logging

---
 .../org/culturegraph/mf/stream/source/FileOpener.java  | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/culturegraph/mf/stream/source/FileOpener.java b/src/main/java/org/culturegraph/mf/stream/source/FileOpener.java
index 9a3760694..208e584be 100644
--- a/src/main/java/org/culturegraph/mf/stream/source/FileOpener.java
+++ b/src/main/java/org/culturegraph/mf/stream/source/FileOpener.java
@@ -29,6 +29,8 @@
 import org.culturegraph.mf.framework.annotations.In;
 import org.culturegraph.mf.framework.annotations.Out;
 import org.culturegraph.mf.util.FileCompression;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 /**
@@ -44,7 +46,8 @@ public final class FileOpener extends DefaultObjectPipe<String, ObjectReceiver<R
 
 	private String encoding = "UTF-8";
 	private FileCompression compression = FileCompression.AUTO;
-
+	private static final Logger LOG = LoggerFactory
+			.getLogger(FileOpener.class);
 	/**
 	 * Returns the encoding used to open the resource.
 	 * 
@@ -101,7 +104,10 @@ public void process(final String file) {
 				throw e;
 			}
 		} catch (IOException e) {
-			throw new MetafactureException(e);
+			LOG.warn(e+"\nCaused by processing file: "+file);
+		}
+		catch (MetafactureException e) {
+			LOG.warn(e+"\nCaused by processing file: "+file);
 		}
 	}
 

From da7e2ef7ca2aa61cbde86bd6e781fe6fe1e6ffad Mon Sep 17 00:00:00 2001
From: pascal Christoph <christoph@hbz-nrw.de>
Date: Fri, 31 Jan 2014 16:08:28 +0100
Subject: [PATCH 2/2] Move catch exception from FileOpener to DirReader

* Add test
* Revert catch exception in FileOpener
* Add catch exception in DirReader
---
 .../mf/stream/source/DirReader.java           | 11 +++-
 .../mf/stream/source/FileOpener.java          | 10 +---
 .../mf/stream/source/DirReaderTest.java       | 51 +++++++++++++++++++
 3 files changed, 63 insertions(+), 9 deletions(-)
 create mode 100644 src/test/java/org/culturegraph/mf/stream/source/DirReaderTest.java

diff --git a/src/main/java/org/culturegraph/mf/stream/source/DirReader.java b/src/main/java/org/culturegraph/mf/stream/source/DirReader.java
index bd40a364b..91ea75331 100644
--- a/src/main/java/org/culturegraph/mf/stream/source/DirReader.java
+++ b/src/main/java/org/culturegraph/mf/stream/source/DirReader.java
@@ -17,11 +17,13 @@
 
 import java.io.File;
 
+import org.culturegraph.mf.exceptions.MetafactureException;
 import org.culturegraph.mf.framework.DefaultObjectPipe;
 import org.culturegraph.mf.framework.ObjectReceiver;
 import org.culturegraph.mf.framework.annotations.Description;
 import org.culturegraph.mf.framework.annotations.In;
 import org.culturegraph.mf.framework.annotations.Out;
+import org.slf4j.LoggerFactory;
 
 /**
  * Reads a directory and emits all filenames found.
@@ -58,7 +60,14 @@ private void dir(final File dir) {
 					dir(file);
 				}
 			} else {
-				receiver.process(file.getAbsolutePath());
+				try {
+					receiver.process(file.getAbsolutePath());
+				}
+				catch (MetafactureException e) {
+					LoggerFactory.getLogger(DirReader.class)
+					.error("Problems with file '"+file+"'", e);
+				}
+				
 			}
 		}
 	}
diff --git a/src/main/java/org/culturegraph/mf/stream/source/FileOpener.java b/src/main/java/org/culturegraph/mf/stream/source/FileOpener.java
index 208e584be..9a3760694 100644
--- a/src/main/java/org/culturegraph/mf/stream/source/FileOpener.java
+++ b/src/main/java/org/culturegraph/mf/stream/source/FileOpener.java
@@ -29,8 +29,6 @@
 import org.culturegraph.mf.framework.annotations.In;
 import org.culturegraph.mf.framework.annotations.Out;
 import org.culturegraph.mf.util.FileCompression;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 
 /**
@@ -46,8 +44,7 @@ public final class FileOpener extends DefaultObjectPipe<String, ObjectReceiver<R
 
 	private String encoding = "UTF-8";
 	private FileCompression compression = FileCompression.AUTO;
-	private static final Logger LOG = LoggerFactory
-			.getLogger(FileOpener.class);
+
 	/**
 	 * Returns the encoding used to open the resource.
 	 * 
@@ -104,10 +101,7 @@ public void process(final String file) {
 				throw e;
 			}
 		} catch (IOException e) {
-			LOG.warn(e+"\nCaused by processing file: "+file);
-		}
-		catch (MetafactureException e) {
-			LOG.warn(e+"\nCaused by processing file: "+file);
+			throw new MetafactureException(e);
 		}
 	}
 
diff --git a/src/test/java/org/culturegraph/mf/stream/source/DirReaderTest.java b/src/test/java/org/culturegraph/mf/stream/source/DirReaderTest.java
new file mode 100644
index 000000000..0445bac22
--- /dev/null
+++ b/src/test/java/org/culturegraph/mf/stream/source/DirReaderTest.java
@@ -0,0 +1,51 @@
+/* Copyright 2013 Pascal Christoph, hbz .
+ *  Licensed under the Apache License, Version 2.0 the "License";
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.culturegraph.mf.stream.source;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Reader;
+import java.net.URISyntaxException;
+
+import org.culturegraph.mf.stream.sink.ObjectWriter;
+import org.junit.Test;
+
+/**
+ * Tests {@link DirReader} in combination with {@link DirReader} and compressed
+ * files. Especially, if there are files which are not (properly) compressed,
+ * the other files should be computed nonetheless.
+ * 
+ * @author Pascal Christoph (dr0i)
+ * 
+ */
+public final class DirReaderTest {
+
+	@Test
+	public void testUtf8IsDefaultEncoding() throws IOException {
+		final DirReader dirReader = new DirReader();
+		final FileOpener opener = new FileOpener();
+		opener.setCompression("bzip2");
+		dirReader.setReceiver(opener);
+		opener.setReceiver(new ObjectWriter<Reader>("stdout"));
+		dirReader.setRecursive(true);
+		try {
+			dirReader.process(new File(Thread.currentThread().getContextClassLoader()
+					.getResource("./").toURI()).getAbsolutePath());
+		} catch (URISyntaxException e) {
+			e.printStackTrace();
+		}
+		opener.closeStream();
+	}
+
+}