Project

General

Profile

Revision 786696e3

ID786696e39d330bd3294a4c50ea38aa685b91dba0
Parent a24ec77b
Child 489492fd

Added by Andreas Müller about 6 years ago

merge trunk into cdm3.3 branch

View differences:

.gitattributes
10 10
cdmlib-commons/src/main/java/eu/etaxonomy/cdm/common/DoubleResult.java -text
11 11
cdmlib-commons/src/main/java/eu/etaxonomy/cdm/common/ExcelUtils.java -text
12 12
cdmlib-commons/src/main/java/eu/etaxonomy/cdm/common/FileCopy.java -text
13
cdmlib-commons/src/main/java/eu/etaxonomy/cdm/common/GeneralParser.java -text
13 14
cdmlib-commons/src/main/java/eu/etaxonomy/cdm/common/ResultWrapper.java -text
14 15
cdmlib-commons/src/main/java/eu/etaxonomy/cdm/common/StreamUtils.java -text
15 16
cdmlib-commons/src/main/java/eu/etaxonomy/cdm/common/Tree.java -text
......
31 32
cdmlib-commons/src/main/resources/MUST-EXIST.txt -text
32 33
cdmlib-commons/src/main/resources/log4j.properties -text
33 34
cdmlib-commons/src/test/java/eu/etaxonomy/cdm/common/CdmUtilsTest.java -text
35
cdmlib-commons/src/test/java/eu/etaxonomy/cdm/common/GeneralParserTest.java -text
34 36
cdmlib-commons/src/test/java/eu/etaxonomy/cdm/common/UriUtilsTest.java -text
35 37
cdmlib-commons/src/test/java/eu/etaxonomy/cdm/common/UuidGenerator.java -text
36 38
cdmlib-commons/src/test/java/eu/etaxonomy/cdm/common/XmlHelpTest.java -text
......
65 67
cdmlib-ext/src/main/java/eu/etaxonomy/cdm/ext/openurl/MobotOpenUrlResponseSchemaAdapter.java -text
66 68
cdmlib-ext/src/main/java/eu/etaxonomy/cdm/ext/openurl/MobotOpenUrlServiceWrapper.java -text
67 69
cdmlib-ext/src/main/java/eu/etaxonomy/cdm/ext/openurl/OpenUrlReference.java -text
70
cdmlib-ext/src/main/java/eu/etaxonomy/cdm/ext/scratchpads/ScratchpadsService.java -text
68 71
cdmlib-ext/src/main/java/eu/etaxonomy/cdm/ext/sru/SruServiceWrapper.java -text
69 72
cdmlib-ext/src/main/resources/eu/etaxonomy/cdm/ext/geo/mapServices_v10.xsd -text
70 73
cdmlib-ext/src/test/java/eu/etaxonomy/cdm/ext/bci/BciServiceTest.java -text
......
418 421
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/LookAheadEventReader.java -text
419 422
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupDocumentImport.java -text
420 423
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupDocumentImportNoComponent.java -text
424
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupFeatureImport.java -text
421 425
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupImportBase.java -text
422 426
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupImportConfigurator.java -text
423 427
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupImportState.java -text
424 428
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupInputStream.java -text
425 429
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupKeyImport.java -text
430
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupModsImport.java -text
426 431
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupNomenclatureImport.java -text
427 432
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupSpecimenImport.java -text
428 433
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupTransformer.java -text
......
683 688
cdmlib-model/src/main/java/eu/etaxonomy/cdm/hibernate/UUIDUserType.java -text
684 689
cdmlib-model/src/main/java/eu/etaxonomy/cdm/hibernate/WSDLDefinitionUserType.java -text
685 690
cdmlib-model/src/main/java/eu/etaxonomy/cdm/hibernate/search/AbstractClassBridge.java -text
691
cdmlib-model/src/main/java/eu/etaxonomy/cdm/hibernate/search/AcceptedTaxonBridge.java -text
692
cdmlib-model/src/main/java/eu/etaxonomy/cdm/hibernate/search/ClassInfoBridge.java -text
686 693
cdmlib-model/src/main/java/eu/etaxonomy/cdm/hibernate/search/DateTimeBridge.java -text
687 694
cdmlib-model/src/main/java/eu/etaxonomy/cdm/hibernate/search/DefinedTermBaseClassBridge.java -text
688 695
cdmlib-model/src/main/java/eu/etaxonomy/cdm/hibernate/search/DescriptionBaseClassBridge.java -text
......
2116 2123
cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/pager/impl/DefaultPagerImpl.java -text
2117 2124
cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/pager/impl/StringLabelPagerImpl.java -text
2118 2125
cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/search/CdmMassIndexer.java -text
2126
cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/search/DocumentSearchResult.java -text
2119 2127
cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/search/ICdmMassIndexer.java -text
2120 2128
cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/search/ISearchResultBuilder.java -text
2121 2129
cdmlib-services/src/main/java/eu/etaxonomy/cdm/api/service/search/LuceneMultiSearch.java -text
cdmlib-commons/src/main/java/eu/etaxonomy/cdm/common/GeneralParser.java
1
/**
2
 * 
3
 */
4
package eu.etaxonomy.cdm.common;
5

  
6
import java.util.regex.Pattern;
7

  
8
import org.apache.log4j.Logger;
9

  
10
/**
11
 * @author a.mueller
12
 * @created 2013-Aug-02
13
 *
14
 */
15
public class GeneralParser {
16
	@SuppressWarnings("unused")
17
	private static final Logger logger = Logger.getLogger(GeneralParser.class);
18
	
19
	static String isbnPatternStr = "^(ISBN\\s*)?(\\d-?){9}((\\d-?){3})?(\\d|X)$";
20
	static Pattern isbnPattern;
21
	
22
	/**
23
	 * Checks if a string follows the ISBN pattern (10 and 13 digits). It does not check the 
24
	 * checksum of an isbn (http://en.wikipedia.org/wiki/International_Standard_Book_Number#ISBN-13_check_digit_calculation).
25
	 * We may implement this separately as sometimes additional checksum validation is unwanted.
26
	 * @param isbn
27
	 * @return
28
	 */
29
	public static boolean isIsbn(String isbn){
30
//		if (isbnPattern == null){
31
//			isbnPattern = Pattern.compile(isbnPatternStr);
32
//		}
33
//		Matcher matcher = isbnPattern.matcher(isbn);
34
		if (isbn == null){
35
			return false;
36
		}else{
37
			return isbn.matches(isbnPatternStr);
38
		}
39
		
40
		
41
		
42
	}
43
	
44
}
cdmlib-commons/src/main/java/eu/etaxonomy/cdm/common/monitor/ProgressMonitorWrapper.java
90 90
     * This implementation of a <code>IProgressMonitor</code>
91 91
     * method forwards to the wrapped progress monitor.
92 92
     * Clients may override this method to do additional
93
     * processing.
93
     * processing.<BR>
94
     * As the  {@link IProgressMonitor interface} documentation for 
95
     * {@link IProgressMonitor#internalWorked(double) this method} 
96
     * says the method must not be called by a client. 
97
     * Clients should always use the method </code>worked(int)</code>.
94 98
     *
95 99
     * @see IProgressMonitor#internalWorked(double)
96 100
     */
cdmlib-commons/src/main/java/eu/etaxonomy/cdm/common/monitor/SubProgressMonitor.java
147 147
        sentToParent = 0;
148 148
    }
149 149

  
150
    /* (Intentionally not javadoc'd)
151
     * Implements the internal method <code>IProgressMonitor.internalWorked</code>.
150

  
151
    /* (non-Javadoc)
152
     * @see eu.etaxonomy.cdm.common.monitor.ProgressMonitorWrapper#internalWorked(double)
152 153
     */
154
    @Override
153 155
    public void internalWorked(double work) {
154 156
        if (usedUp || nestedBeginTasks != 1) {
155 157
            return;
cdmlib-commons/src/test/java/eu/etaxonomy/cdm/common/GeneralParserTest.java
1
/**
2
 * 
3
 */
4
package eu.etaxonomy.cdm.common;
5

  
6
import static org.junit.Assert.*;
7

  
8
import org.junit.Assert;
9
import org.junit.Before;
10
import org.junit.Test;
11

  
12
/**
13
 * @author a.mueller
14
 *
15
 */
16
public class GeneralParserTest {
17

  
18
	/**
19
	 * @throws java.lang.Exception
20
	 */
21
	@Before
22
	public void setUp() throws Exception {
23
	}
24

  
25
	@Test
26
	public void test() {
27
		String isbn1 = "ISBN 90-71236-61-7";
28
		Assert.assertTrue("isbn1 should be an ISBN", GeneralParser.isIsbn(isbn1));
29
		
30
		String isbn2 = "90-71236-61-7";
31
		Assert.assertTrue("'ISBN' string should not be required", GeneralParser.isIsbn(isbn2));
32
		
33
		String isbnToShort = "ISBN 90-7123-61-7";
34
		Assert.assertFalse("ISBN must have 10 or 13 numbers, 9 numbered string should not be an ISBN", GeneralParser.isIsbn(isbnToShort));
35
		
36
		String isbn3 = "ISBN 123-456-789-112-3";
37
		Assert.assertTrue("isbn3 (with 13 numbers) should be an ISBN", GeneralParser.isIsbn(isbn3));
38
		
39
		String isbn4 = "ISBN 123-456-789-112-3-";
40
		Assert.assertFalse("- at the end of ISBN is not allowed", GeneralParser.isIsbn(isbn4));
41

  
42
		String isbn5 = "ISBN 123-456-789-12-3";
43
		Assert.assertFalse("12 numbers are not allowed, either 10 or 13", GeneralParser.isIsbn(isbn5));
44
		
45
		String isbn6 = "ISBN 123-456-789-112-X";
46
		Assert.assertTrue("X should be allowed as a final digit", GeneralParser.isIsbn(isbn6));
47

  
48
	}
49

  
50
}
cdmlib-ext/pom.xml
92 92
	    <groupId>org.apache.httpcomponents</groupId>
93 93
	    <artifactId>httpclient</artifactId>
94 94
	</dependency>
95
	<dependency>
96
        <groupId>net.sf.json-lib</groupId>
97
                <artifactId>json-lib</artifactId>
98
				<classifier>jdk15</classifier>
99
    </dependency>
100
	<dependency>
101
        <groupId>net.sf.ezmorph</groupId>
102
        <artifactId>ezmorph</artifactId>
103
    </dependency>
95 104

  
96 105
  </dependencies>
97 106
</project>
cdmlib-ext/src/main/java/eu/etaxonomy/cdm/ext/scratchpads/ScratchpadsService.java
1
// $Id$
2
/**
3
 * Copyright (C) 2009 EDIT
4
 * European Distributed Institute of Taxonomy 
5
 * http://www.e-taxonomy.eu
6
 * 
7
 * The contents of this file are subject to the Mozilla Public License Version 1.1
8
 * See LICENSE.TXT at the top of this package for the full license terms.
9
 */
10
package eu.etaxonomy.cdm.ext.scratchpads;
11

  
12
import java.io.BufferedReader;
13
import java.io.FileNotFoundException;
14
import java.io.FileOutputStream;
15
import java.io.IOException;
16
import java.io.InputStream;
17
import java.io.InputStreamReader;
18
import java.net.URI;
19
import java.net.URISyntaxException;
20
import java.net.URL;
21
import java.util.Iterator;
22
import java.util.zip.ZipEntry;
23
import java.util.zip.ZipInputStream;
24

  
25
import org.apache.http.HttpException;
26
import org.apache.http.HttpResponse;
27
import org.apache.log4j.Logger;
28
import org.codehaus.jackson.JsonNode;
29
import org.codehaus.jackson.map.ObjectMapper;
30

  
31
import eu.etaxonomy.cdm.common.UriUtils;
32

  
33

  
34
/**
35
 * @author l.morris
36
 * @date Jul 2, 2013
37
 *
38
 */
39
public class ScratchpadsService {
40

  
41
	private static final Logger logger = Logger.getLogger(ScratchpadsService.class);
42

  
43
	public static final String SCRATCHPADS_JSON_ENDPOINT = "http://scratchpads.eu/explore/sites-list/json";
44

  
45
	private static final char[] ILLEGAL_CHARACTERS = { '/', '\n', '\r', '\t', '\0', '\f', '`', '?', '*', '\\', '<', '>', '|', '\"', ':', '.' };
46

  
47
	private static final String dir = "C:\\Users\\l.morris\\Downloads\\dwca_scratchpads\\";
48
	
49
	public void harvest(){
50

  
51
		InputStream inputStream = null;
52
		
53
		try {
54
			URL url = new URL(SCRATCHPADS_JSON_ENDPOINT);
55
			boolean isAvailable = UriUtils.isServiceAvailable(url.toURI());
56

  
57
			if (isAvailable) {
58
				inputStream = UriUtils.getInputStream(url.toURI());
59
			}
60

  
61
			BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, "iso-8859-1"), 8);
62
			StringBuilder sb = new StringBuilder();
63
			String line = null;
64
			while ((line = reader.readLine()) != null) {
65
				sb.append(line + "\n");
66
				logger.debug(line);
67
				
68
			}
69

  
70
			ObjectMapper m = new ObjectMapper();			
71
			
72
			JsonNode rootNode = m.readTree(sb.toString());
73
			logger.debug(rootNode.toString());
74
			logger.debug(rootNode.isArray());
75
			
76
			int num = 0;
77
			if (rootNode.isArray()) {
78
				Iterator<JsonNode> arrayElements = rootNode.getElements();
79
				while (arrayElements.hasNext()) {
80
					JsonNode element = arrayElements.next();
81
					JsonNode website = element.path("field_website");
82
					//logger.debug(website.getValueAsText());
83
					String fieldWebsite = website.getValueAsText();
84
					
85
					if (fieldWebsite.startsWith("http")) {
86

  
87
						url = new URL(fieldWebsite + "/dwca.zip");
88
						URI uri = url.toURI();
89
						isAvailable = UriUtils.isServiceAvailable(uri);
90
						logger.debug("Is " + fieldWebsite + " available :" + isAvailable);
91

  
92
						String websiteName = "";
93
						//websiteName = (fieldWebsite.toString().split("//")[1]).split(".*")[0];
94
						websiteName = websiteName + fieldWebsite.split("//")[1];
95
						//if (websiteName.contains(".")){
96
							//websiteName = websiteName.substring(0, websiteName.indexOf("."));
97
						websiteName = websiteName.replaceAll("\\.", "_");
98
							//websiteName = websiteName.substring(0, websiteName.indexOf("."));
99

  
100
						//} 
101

  
102
						//logger.debug("the website name " + websiteName);
103

  
104
						for (int j = 0; j < ILLEGAL_CHARACTERS.length; j++) {
105

  
106
							char ch = '_';
107
							websiteName.replace(ILLEGAL_CHARACTERS[j], ch);
108
						}
109

  
110
						websiteName = websiteName.substring(0, websiteName.length());
111

  
112
						if (isAvailable) {
113

  
114
							HttpResponse response = UriUtils.getResponse(uri, null);
115
							if (UriUtils.isOk(response)) {
116

  
117

  
118
								logger.debug("There is a dwca " + websiteName);
119

  
120
								try {
121
									inputStream = UriUtils.getInputStream(url.toURI());
122

  
123
									num++;
124

  
125
									if (inputStream != null) {
126

  
127
										copyDwcaZip(inputStream, websiteName);
128
										//createDwcaZip(inputStream);
129
									}
130
									
131
								} catch (HttpException e) {
132
									// TODO Auto-generated catch block
133
									logger.error("Failed to get dwca for " + websiteName + " as there was an error " + e);
134
								} 
135

  
136
							}
137

  
138
						}
139
					}
140
				}
141
			}
142

  
143
			inputStream.close();
144

  
145

  
146
		} catch (URISyntaxException e) {
147
			// TODO Auto-generated catch block
148
			e.printStackTrace();
149
		} catch (IOException ie) {
150
			ie.printStackTrace();
151
		} catch (HttpException e) {
152
			// TODO Auto-generated catch block
153
			e.printStackTrace();
154
		} 
155
	}
156
	
157
	/**
158
	 * FIXME
159
	 * This is a hack as dwca.zip files from Scratchpads sites have an extra directory when unzipped. i.e. all the text
160
	 * and meta.xml are in the sub-directory dwca, but the should be directly in the top-level unzipped directory
161
	 */
162
	private void createDwcaZip (InputStream inputStream, String websiteName) {
163
		
164
		 ZipInputStream zis = new ZipInputStream(inputStream);
165
	    
166
	         byte[] buffer = new byte[4096];
167
	         ZipEntry ze;
168
	         try {
169
				while ((ze = zis.getNextEntry()) != null)
170
				 {
171
				    System.out.println("Extracting: " + ze);
172
				    
173
				    FileOutputStream fos = new FileOutputStream(ze.getName());
174
				    {
175
				       int numBytes;
176
				       while ((numBytes = zis.read(buffer, 0, buffer.length)) != -1)
177
				          fos.write(buffer, 0, numBytes);
178
				    }
179
				    zis.closeEntry();
180
				 }
181
			} catch (FileNotFoundException e) {
182
				// TODO Auto-generated catch block
183
				e.printStackTrace();
184
			} catch (IOException e) {
185
				// TODO Auto-generated catch block
186
				e.printStackTrace();
187
			}		
188
		
189
	}
190
	
191
	/*
192
	 * Use this method instead of createDwcaZip, once the dwca.zip structure is fixed in Scratchpads
193
	 */
194
	private void copyDwcaZip (InputStream inputStream, String websiteName) {
195
		
196
		FileOutputStream outputStream;
197
		try {
198
			outputStream = new FileOutputStream("dwca_" + websiteName + ".zip");//dir + 
199

  
200
			byte[] b = new byte[1024];
201
			int count;
202
			while ((count = inputStream.read(b)) >= 0) {
203
				outputStream.write(b, 0, count);
204
			}
205
			outputStream.flush(); 
206
			outputStream.close(); 
207
			inputStream.close();   
208
		} catch (FileNotFoundException e) {
209
			// TODO Auto-generated catch block
210
			e.printStackTrace();
211
		} catch (IOException ie) {
212
			ie.printStackTrace();
213
		} 
214
	}
215

  
216
	/**
217
	 * @param args
218
	 */
219
	public static void main(String[] args) {
220

  
221
		ScratchpadsService spService = new ScratchpadsService();
222
		spService.harvest();
223
		// TODO Auto-generated method stub
224

  
225
	}
226

  
227
}
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/ImportStateBase.java
23 23
import eu.etaxonomy.cdm.model.common.AnnotationType;
24 24
import eu.etaxonomy.cdm.model.common.CdmBase;
25 25
import eu.etaxonomy.cdm.model.common.DefinedTerm;
26
import eu.etaxonomy.cdm.model.common.DefinedTermBase;
26 27
import eu.etaxonomy.cdm.model.common.ExtensionType;
27 28
import eu.etaxonomy.cdm.model.common.Language;
28 29
import eu.etaxonomy.cdm.model.common.MarkerType;
......
105 106
//			}
106 107
			setTransformer(newTransformer);
107 108
		}
109
		
108 110
	}
109 111
	
112
	
113
	
114
	/**
115
	 * Resets (empties) all maps which map a uuid to a {@link DefinedTermBase term}.
116
	 * This is usually needed when a a new transaction is opened and user defined terms are reused.
117
	 */
118
	public void resetUuidTermMaps(){
119
		extensionTypeMap = new HashMap<UUID, ExtensionType>();
120
		markerTypeMap = new HashMap<UUID, MarkerType>();
121
		annotationTypeMap = new HashMap<UUID, AnnotationType>();
122
		
123
		namedAreaMap = new HashMap<UUID, NamedArea>();
124
		namedAreaLevelMap = new HashMap<UUID, NamedAreaLevel>();
125
		featureMap = new HashMap<UUID, Feature>();
126
		stateTermMap = new HashMap<UUID, State>();
127
		measurementUnitMap = new HashMap<UUID, MeasurementUnit>();
128
		statisticalMeasureMap = new HashMap<UUID, StatisticalMeasure>();
129
		modifierMap = new HashMap<UUID, DefinedTerm>();
130
		
131
		presenceTermMap = new HashMap<UUID, PresenceTerm>();;
132
		languageMap = new HashMap<UUID, Language>();
133
		taxonRelationshipTypeMap = new HashMap<UUID, TaxonRelationshipType>();
134
		
135
		referenceSystemMap = new HashMap<UUID, ReferenceSystem>();
136
		rankMap = new HashMap<UUID, Rank>();
137
	}
138
	
139
	
110 140
	//different type of stores that are used by the known imports
111 141
	protected Map<String, MapWrapper<? extends CdmBase>> stores = new HashMap<String, MapWrapper<? extends CdmBase>>();
112 142
	
......
169 199
	}
170 200

  
171 201
	
172
	
173
	
174 202
	/**
175 203
	 * Adds a classification uuid to the classification uuid map,
176 204
	 * which maps a key for the classification to its UUID in the CDM
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/XmlImportState.java
11 11
package eu.etaxonomy.cdm.io.common;
12 12

  
13 13
import javax.xml.stream.XMLEventReader;
14
import javax.xml.stream.XMLStreamException;
15
import javax.xml.stream.events.XMLEvent;
16 14

  
17 15
import org.apache.log4j.Logger;
18 16

  
19 17
/**
20 18
 * @author a.mueller
21 19
 * @created 11.05.2009
22
 * @version 1.0
23 20
 */
24 21
public class XmlImportState<CONFIG extends XmlImportConfiguratorBase, IO extends XmlImportBase> extends ImportStateBase<CONFIG, IO> {
25 22
	@SuppressWarnings("unused")
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/dwca/TermUri.java
38 38
	DC_LICENSE("http://purl.org/dc/terms/license"),
39 39
	DC_MODIFIED("http://purl.org/dc/terms/modified"),
40 40
	DC_PUBLISHER("http://purl.org/dc/terms/publisher"),
41
	DC_REFERENCES("http://purl.org/dc/terms/references"),
41 42
	DC_RIGHTS("http://purl.org/dc/terms/rights"),
42 43
	DC_RIGHTS_HOLDER("http://purl.org/dc/terms/rightsHolder"),
43 44
	DC_SOURCE("http://purl.org/dc/terms/source"),
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/dwca/in/DwcTaxonStreamItem2CdmTaxonConverter.java
9 9
*/
10 10
package eu.etaxonomy.cdm.io.dwca.in;
11 11

  
12
import java.net.URI;
12 13
import java.util.ArrayList;
13 14
import java.util.HashSet;
14 15
import java.util.List;
......
27 28
import eu.etaxonomy.cdm.io.stream.StreamItem;
28 29
import eu.etaxonomy.cdm.model.common.Annotation;
29 30
import eu.etaxonomy.cdm.model.common.CdmBase;
31
import eu.etaxonomy.cdm.model.common.Extension;
32
import eu.etaxonomy.cdm.model.common.ExtensionType;
30 33
import eu.etaxonomy.cdm.model.common.IdentifiableSource;
31 34
import eu.etaxonomy.cdm.model.common.LSID;
32 35
import eu.etaxonomy.cdm.model.common.Language;
......
58 61
 *
59 62
 */
60 63
public class  DwcTaxonStreamItem2CdmTaxonConverter<CONFIG extends DwcaDataImportConfiguratorBase, STATE extends StreamImportStateBase<CONFIG, StreamImportBase>>  extends PartitionableConverterBase<CONFIG, STATE> implements IPartitionableConverter<StreamItem, IReader<CdmBase>, String>{
61
	@SuppressWarnings("unused")
62
	private static Logger logger = Logger.getLogger(DwcTaxonStreamItem2CdmTaxonConverter.class);
64
	private static final Logger logger = Logger.getLogger(DwcTaxonStreamItem2CdmTaxonConverter.class);
63 65

  
64 66
	private static final String ID = "id";
65 67
	// temporary key for the case that no dataset information is supplied, TODO use something better
......
104 106
		
105 107
		//nameAccordingTo
106 108
		MappedCdmBase<Reference> sec = getNameAccordingTo(csvTaxonRecord, resultList);
109
		
107 110
		if (sec == null && state.getConfig().isUseSourceReferenceAsSec()){
108 111
			sec = new MappedCdmBase<Reference>(state.getTransactionalSourceReference());
109 112
		}
......
187 190
	 * @param taxonBase
188 191
	 */
189 192
	private void handleIdentifiableObjects(StreamItem item,TaxonBase<?> taxonBase) {
193
		
194
		
195
		String references = item.get(TermUri.DC_REFERENCES);
196
		
197
		if (references == null || references == "") {
198
			references = item.get(TermUri.DWC_NAME_PUBLISHED_IN_ID);//lorna temporary until Scratchpads move the reference to the correct place.
199
		}
200
		
201
		if (StringUtils.isNotBlank(references)){
202
			URI uri = makeUriIfIs(references);
203
			if (uri != null){
204
				Extension.NewInstance(taxonBase, references, ExtensionType.URL());
205
			}else{
206
				String message = "Non-URI Dublin Core References not yet handled for taxa. References is: %s";
207
				fireWarningEvent(String.format(message, references), item, 6);
208
			}
209
		}
210
		
211
		
190 212
		//TODO: Finish properly
191 213
		String id = item.get(TermUri.CDM_SOURCE_IDINSOURCE);
192 214
		String idNamespace = item.get(TermUri.CDM_SOURCE_IDNAMESPACE);
......
197 219
			Taxon taxon = (Taxon) taxonBase;
198 220
			taxon.addSource(OriginalSourceType.Import, id, idNamespace, ref, null);
199 221
		}
222
		
223
		
224
		
225
	}
226

  
227

  
228
	/**
229
	 * If str is an uri it returns is as an {@link URI}. If not it returns <code>null</code>. 
230
	 * @param str
231
	 * @return the URI.
232
	 */
233
	private URI makeUriIfIs(String str) {
234
		if (! str.startsWith("http:")){
235
			return null;
236
		}else{
237
			try {
238
				URI uri = URI.create(str);
239
				return uri;
240
			} catch (Exception e) {
241
				return null;
242
			}
243
		}
244

  
200 245
	}
201 246

  
202 247

  
......
226 271
	private void handleTdwgArea(StreamItem item, TaxonBase<?> taxonBase) {
227 272
		// TODO Auto-generated method stub
228 273
		String tdwg_area = item.get(TermUri.DWC_COUNTRY_CODE);
274
		if (tdwg_area != null){
229 275
		if(taxonBase instanceof Synonym){
230 276
			Synonym synonym = CdmBase.deproxy(taxonBase, Synonym.class);
231 277
			Set<Taxon> acceptedTaxaList = synonym.getAcceptedTaxa();
......
236 282
				for(Taxon taxon : acceptedTaxaList){
237 283
					TaxonDescription td = getTaxonDescription(taxon, false);
238 284
					NamedArea area = NamedArea.getAreaByTdwgAbbreviation(tdwg_area);
239

  
285
	
240 286
					if (area == null){
241 287
						area = NamedArea.getAreaByTdwgLabel(tdwg_area);
242 288
					}
......
251 297
			Taxon taxon = CdmBase.deproxy(taxonBase, Taxon.class);
252 298
			TaxonDescription td = getTaxonDescription(taxon, false);
253 299
			NamedArea area = NamedArea.getAreaByTdwgAbbreviation(tdwg_area);
254

  
300
	
255 301
			if (area == null){
256 302
				area = NamedArea.getAreaByTdwgLabel(tdwg_area);
257 303
			}
......
261 307
			}
262 308
		}
263 309
	}
310
	}
264 311

  
265 312

  
266 313
	/**
......
327 374
			//if not exists, create new
328 375
			if (! classificationExists){
329 376
				String classificationName = StringUtils.isBlank(datasetName)? datasetId : datasetName;
330
				if (classificationName.equals(NO_DATASET)){
331
					classificationName = "Classification (no name)";  //TODO define by config or zipfile or metadata
377
				if (classificationName.equals(NO_DATASET)){					
378
					classificationName = config.getClassificationName();
379
					//classificationName = "Classification (no name)";  //TODO define by config or zipfile or metadata
332 380
				}
333 381
				
334 382
				String classificationId = StringUtils.isBlank(datasetId)? datasetName : datasetId;
......
497 545
					if (! idIsInternal){
498 546
						//references should already exist in store if not linking to external links like URLs
499 547
						String message = "External namePublishedInIDs are not yet supported";
500
						fireWarningEvent(message, item, 4);
548
						fireWarningEvent(message, item, 4);//set to DEBUG
501 549
					}else{
502 550
						newRef = ReferenceFactory.newGeneric();  //TODO handle other types if possible
503 551
						newRef.addSource(OriginalSourceType.Import, refId, idTerm.toString(), sourceCitation, null);
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/dwca/in/DwcaZipToStreamConverter.java
18 18
import java.net.URI;
19 19
import java.util.ArrayList;
20 20
import java.util.Arrays;
21
import java.util.Enumeration;
21 22
import java.util.HashMap;
22 23
import java.util.List;
23 24
import java.util.Map;
......
263 264
			throw new IOException(String.format(message, dwcaZip.toString()));
264 265
		}
265 266
		ZipFile zip = new ZipFile(file, ZipFile.OPEN_READ);
267
		
268
		//Enumeration<? extends ZipEntry> zipEntries = zip.entries();
269
		//ze = new ZipEntry(name);
266 270
		ZipEntry metaEntry = zip.getEntry(name);
271
		
272
		//Lorna added this to deal with Scratchpads dwca.zip files which when unzipped have a directory dwca/ which contains the files
273
		if (metaEntry == null) {
274
			metaEntry = zip.getEntry("dwca/" + name);
275
		}
267 276
		if (metaEntry == null){
268 277
			String message = "Zip entry for %s not available";
269 278
			throw new IOException(String.format(message, name));
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/dwca/in/GbifDescriptionCsv2CdmConverter.java
27 27
import eu.etaxonomy.cdm.model.description.Feature;
28 28
import eu.etaxonomy.cdm.model.description.TaxonDescription;
29 29
import eu.etaxonomy.cdm.model.description.TextData;
30
import eu.etaxonomy.cdm.model.media.Rights;
30 31
import eu.etaxonomy.cdm.model.reference.Reference;
31 32
import eu.etaxonomy.cdm.model.taxon.Taxon;
32 33

  
......
62 63
		if (taxon != null){
63 64
			
64 65
			String description = item.get(TermUri.DC_DESCRIPTION);
66
			//String license = item.get(TermUri.DC_LICENSE);//lorna check - often empty in SP dwca
67
					
68
			//TODO: Create the Language from the TermUri.DC_LANGUAGE in the dwca
69
			Language language = getLanguage(item); 
70
									
71
			
65 72
			if (StringUtils.isNotBlank(description)){
66 73
				Feature feature = getFeatureByDcType(item, resultList);
67 74

  
68 75
				TaxonDescription taxonDescription = getTaxonDescription(taxon, false);
76
								
77
				//deduplicate - if this taxonDescription already got these Rights attached
78
				boolean addRights = true;
79
				String rights = item.get(TermUri.DC_RIGHTS);
80
				Set<Rights> allRights = taxonDescription.getRights();				
81
				for (Rights r : allRights) {					
82
					if (r.getText() == rights) {
83
						addRights = false;
84
					}
85
				}
86
				
87
				if (addRights && (rights != null || rights != "")) {
88
					Rights copyright = Rights.NewInstance(rights, language);
89
					taxonDescription.addRights(copyright);
90
				}
91
				
69 92
				TextData descElement = TextData.NewInstance(feature);
70 93
				
71
				Language language = getLanguage(item);  //TODO
94
				//Language language = getLanguage(item);  //TODO
72 95
				descElement.putText(language,description);
73 96
				taxonDescription.addElement(descElement);
74 97
			}else{
......
92 115
		//TODO
93 116
		
94 117
		Language language = Language.DEFAULT();
118
		String langString = item.get(TermUri.DC_LANGUAGE);	
119
		
120
		/*if (langString != null) {
121
			if (!langString.equals("")) {
122
				language = getTermService().getLanguageByIso(langString.substring(0, 2));
123
				//can getTermService from StreamImportBase which calls GbifDescriptionCsv2CdmConverter.map
124
			}
125
		}*/
95 126
		return language;
96 127
	}
97 128

  
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/FeatureSorter.java
9 9
import java.util.HashSet;
10 10
import java.util.Iterator;
11 11
import java.util.List;
12
import java.util.ListIterator;
13 12
import java.util.Map;
14 13
import java.util.Set;
15 14
import java.util.UUID;
16 15

  
17 16
/**
18 17
 * This class is supposed to find the best sorting order for features (descriptive and other).
18
 * Currently it is not yet very sophisticated.
19
 * 
19 20
 * @author a.mueller
20 21
 *
21 22
 */
......
62 63
			Map<UUID,FeatureStatistic> statisticMap = computeStatistic(listMap);
63 64
			FeatureStatistic best = findBest(statisticMap);
64 65
			result.add(best.uuid);
65
			removeFromLists(listMap, best.uuid);
66
			Map<String, List<FeatureSorterInfo>> subFeatures = removeFromLists(listMap, best.uuid);
67
			List<UUID> subFeatureOrder = getSortOrder(subFeatures);
68
			result.addAll(subFeatureOrder);
66 69
		}
67 70
		return result;
68 71
	}
......
88 91
	 * @param orderLists
89 92
	 * @param uuid
90 93
	 */
91
	private void removeFromLists(Map<String, List<FeatureSorterInfo>> orderLists, UUID uuid) {
94
	private Map<String, List<FeatureSorterInfo>> removeFromLists(Map<String, List<FeatureSorterInfo>> orderListsMap, UUID uuid) {
95
		Map<String, List<FeatureSorterInfo>> childLists = new HashMap<String, List<FeatureSorterInfo>>();
92 96
		
93
		Set<String> keySet = orderLists.keySet();
97
		Set<String> keySet = orderListsMap.keySet();
94 98
		Iterator<String> keySetIterator = keySet.iterator();
95 99
		while (keySetIterator.hasNext()){
96 100
			String key = keySetIterator.next();
97
			List<FeatureSorterInfo> list = orderLists.get(key);			
101
			List<FeatureSorterInfo> list = orderListsMap.get(key);			
98 102
			Iterator<FeatureSorterInfo> it = list.listIterator();
99 103
			while (it.hasNext()){
100 104
				FeatureSorterInfo info = it.next();
101 105
				if (info.getUuid().equals(uuid)){
106
					if (! info.getSubFeatures().isEmpty()){
107
						childLists.put(key, info.getSubFeatures());
108
					}
102 109
					it.remove();
103 110
					if (list.isEmpty()){
104 111
						keySetIterator.remove();
......
106 113
				}
107 114
			}
108 115
		}
116
		return childLists;
109 117
	}
110 118

  
111 119

  
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/FeatureSorterInfo.java
3 3
 */
4 4
package eu.etaxonomy.cdm.io.markup;
5 5

  
6
import java.util.ArrayList;
7
import java.util.List;
6 8
import java.util.UUID;
7 9

  
8 10
import org.apache.commons.lang.StringUtils;
......
18 20

  
19 21
	private UUID uuid;
20 22
	private String title;
23
	private List<FeatureSorterInfo> subFeatures = new ArrayList<FeatureSorterInfo>();
21 24

  
22 25
	public FeatureSorterInfo(Feature feature){
23 26
		this.setUuid(feature.getUuid());
......
35 38
	private void setUuid(UUID uuid) {
36 39
		this.uuid = uuid;
37 40
	}
41
	
42
	public List<FeatureSorterInfo> getSubFeatures() {
43
		return subFeatures;
44
	}
45
	
46
	public void addSubFeature(FeatureSorterInfo subFeature){
47
		subFeatures.add(subFeature);
48
	}
49
	
38 50
	@Override
39 51
	public String toString(){
40 52
		if (StringUtils.isNotBlank(title)){
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupDocumentImport.java
76 76
	public void doInvoke(MarkupImportState state) { 
77 77
		fireProgressEvent("Start import markup document", "Before start of document");
78 78
		
79
		Queue<CdmBase> outputStream = new LinkedList<CdmBase>();
80

  
81 79
		TransactionStatus tx = startTransaction();
82 80
		// FIXME reset state
83 81
		doAllTheOldOtherStuff(state);
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/markup/MarkupDocumentImportNoComponent.java
11 11

  
12 12
import java.net.MalformedURLException;
13 13
import java.net.URL;
14
import java.util.ArrayList;
15
import java.util.Arrays;
16
import java.util.HashMap;
17 14
import java.util.HashSet;
18
import java.util.List;
19 15
import java.util.Map;
20 16
import java.util.Set;
21 17
import java.util.UUID;
22
import java.util.regex.Matcher;
23
import java.util.regex.Pattern;
24 18

  
25 19
import javax.xml.stream.Location;
26 20
import javax.xml.stream.XMLEventReader;
......
32 26
import org.apache.commons.lang.StringUtils;
33 27
import org.apache.log4j.Logger;
34 28

  
35
import eu.etaxonomy.cdm.common.CdmUtils;
36 29
import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
37 30
import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
38
import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
39
import eu.etaxonomy.cdm.model.common.Annotation;
40
import eu.etaxonomy.cdm.model.common.AnnotationType;
41
import eu.etaxonomy.cdm.model.common.CdmBase;
42
import eu.etaxonomy.cdm.model.common.Extension;
43 31
import eu.etaxonomy.cdm.model.common.ExtensionType;
44 32
import eu.etaxonomy.cdm.model.common.Language;
45 33
import eu.etaxonomy.cdm.model.common.OriginalSourceType;
......
50 38
import eu.etaxonomy.cdm.model.description.Feature;
51 39
import eu.etaxonomy.cdm.model.description.PolytomousKey;
52 40
import eu.etaxonomy.cdm.model.description.PolytomousKeyNode;
53
import eu.etaxonomy.cdm.model.description.PresenceAbsenceTermBase;
54
import eu.etaxonomy.cdm.model.description.PresenceTerm;
55 41
import eu.etaxonomy.cdm.model.description.TaxonDescription;
56 42
import eu.etaxonomy.cdm.model.description.TextData;
57
import eu.etaxonomy.cdm.model.location.NamedArea;
58
import eu.etaxonomy.cdm.model.location.NamedAreaLevel;
59
import eu.etaxonomy.cdm.model.media.IdentifiableMediaEntity;
60
import eu.etaxonomy.cdm.model.media.Media;
61 43
import eu.etaxonomy.cdm.model.name.CultivarPlantName;
62 44
import eu.etaxonomy.cdm.model.name.NonViralName;
63 45
import eu.etaxonomy.cdm.model.name.Rank;
64
import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
65 46
import eu.etaxonomy.cdm.model.reference.Reference;
66 47
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
67 48
import eu.etaxonomy.cdm.model.taxon.Classification;
68 49
import eu.etaxonomy.cdm.model.taxon.Taxon;
69 50
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
70
import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
71 51

  
72 52

  
73 53
/**
......
75 55
 * 
76 56
 */
77 57
public class MarkupDocumentImportNoComponent extends MarkupImportBase {
58
	@SuppressWarnings("unused")
78 59
	private static final Logger logger = Logger.getLogger(MarkupDocumentImportNoComponent.class);
79 60
	
80 61
	private MarkupKeyImport keyImport;
81
	private MarkupSpecimenImport specimenImport;
82 62

  
63
	private MarkupModsImport modsImport;
64
	private MarkupFeatureImport featureImport;
65
	private MarkupSpecimenImport specimenImport;
83 66
	private MarkupNomenclatureImport nomenclatureImport;
84
	
67

  
85 68
	public MarkupDocumentImportNoComponent(MarkupDocumentImport docImport) {
86 69
		super(docImport);
87 70
		this.keyImport = new MarkupKeyImport(docImport);
88 71
		this.specimenImport = new MarkupSpecimenImport(docImport);
89
		nomenclatureImport = new MarkupNomenclatureImport(docImport, keyImport, specimenImport);
72
		this.nomenclatureImport = new MarkupNomenclatureImport(docImport, specimenImport);
73
		this.modsImport = new MarkupModsImport(docImport);
74
		this.featureImport = new MarkupFeatureImport(docImport, specimenImport, nomenclatureImport);
90 75
	}
91 76

  
92 77
	public void doInvoke(MarkupImportState state) throws XMLStreamException { 
......
193 178
					message = String.format(message, baseUrl);
194 179
					fireWarningEvent(message, next, 8);
195 180
				}
196
			} else if (isStartingElement(next, MODS)){
197
				handleNotYetImplementedElement(next);
181
			} else if (isStartingElement(next, MODS)){	
182
				modsImport.handleMods(state, reader, next);
198 183
			} else {
199 184
				handleUnexpectedElement(next);
200 185
			}
......
398 383
					nomenclatureImport.handleNomenclature(state, reader, next);
399 384
					hasNomenclature = true;
400 385
				} else if (isStartingElement(next, FEATURE)) {
401
					handleFeature(state, reader, next);
386
					featureImport.handleFeature(state, reader, next);
402 387
				} else if (isStartingElement(next, NOTES)) {
403 388
					// TODO is this the correct way to handle notes?
404 389
					String note = handleNotes(state, reader, next);
......
425 410
						desc.addElement(textData);
426 411
					}
427 412
					textData = (TextData)desc.getElements().iterator().next();
428
					makeFeatureFigureRef(state, reader, desc, false, textData, next);
413
					featureImport.makeFeatureFigureRef(state, reader, desc, false, textData, next);
429 414
				} else if (isStartingElement(next, FIGURE)) {
430
					handleFigure(state, reader, next);
415
					handleFigure(state, reader, next, specimenImport, nomenclatureImport);
431 416
				} else if (isStartingElement(next, FOOTNOTE)) {
432
					FootnoteDataHolder footnote = handleFootnote(state, reader,	next);
417
					FootnoteDataHolder footnote = handleFootnote(state, reader,	next, specimenImport, nomenclatureImport);
433 418
					if (footnote.isRef()) {
434 419
						String message = "Ref footnote not implemented here";
435 420
						fireWarningEvent(message, next, 4);
......
458 443
		WriterDataHolder writer = handleWriter(state, reader, next);
459 444
		taxon.addExtension(writer.extension);
460 445
		// TODO what if taxonTitle comes later
461
		if (StringUtils.isNotBlank(taxonTitle)
462
				&& writer.extension != null) {
446
		if (StringUtils.isNotBlank(taxonTitle) && writer.extension != null) {
463 447
			Reference<?> sec = ReferenceFactory.newBookSection();
464 448
			sec.setTitle(taxonTitle);
465 449
			TeamOrPersonBase<?> author = createAuthor(writer.writer);
466 450
			sec.setAuthorTeam(author);
467
			sec.setInReference(state.getConfig()
468
					.getSourceReference());
451
			sec.setInReference(state.getConfig().getSourceReference());
469 452
			taxon.setSec(sec);
470 453
			registerFootnotes(state, sec, writer.footnotes);
471 454
		} else {
......
645 628

  
646 629
	}
647 630

  
648
	private WriterDataHolder handleWriter(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
649
		String text = "";
650
		checkNoAttributes(parentEvent);
651
		WriterDataHolder dataHolder = new WriterDataHolder();
652
		List<FootnoteDataHolder> footnotes = new ArrayList<FootnoteDataHolder>();
653

  
654
		// TODO handle attributes
655
		while (reader.hasNext()) {
656
			XMLEvent next = readNoWhitespace(reader);
657
			if (isMyEndingElement(next, parentEvent)) {
658
				text = CdmUtils.removeBrackets(text);
659
				if (checkMandatoryText(text, parentEvent)) {
660
					text = normalize(text);
661
					dataHolder.writer = text;
662
					dataHolder.footnotes = footnotes;
663

  
664
					// Extension
665
					UUID uuidWriterExtension = MarkupTransformer.uuidWriterExtension;
666
					ExtensionType writerExtensionType = this
667
							.getExtensionType(state, uuidWriterExtension,
668
									"Writer", "writer", "writer");
669
					Extension extension = Extension.NewInstance();
670
					extension.setType(writerExtensionType);
671
					extension.setValue(text);
672
					dataHolder.extension = extension;
673

  
674
					// Annotation
675
					UUID uuidWriterAnnotation = MarkupTransformer.uuidWriterAnnotation;
676
					AnnotationType writerAnnotationType = this.getAnnotationType(state, uuidWriterAnnotation, "Writer", "writer", "writer", null);
677
					Annotation annotation = Annotation.NewInstance(text, writerAnnotationType, getDefaultLanguage(state));
678
					dataHolder.annotation = annotation;
679

  
680
					return dataHolder;
681
				} else {
682
					return null;
683
				}
684
			} else if (isStartingElement(next, FOOTNOTE_REF)) {
685
				FootnoteDataHolder footNote = handleFootnoteRef(state, reader, next);
686
				if (footNote.isRef()) {
687
					footnotes.add(footNote);
688
				} else {
689
					logger.warn("Non ref footnotes not yet impelemnted");
690
				}
691
			} else if (next.isCharacters()) {
692
				text += next.asCharacters().getData();
693

  
694
			} else {
695
				handleUnexpectedElement(next);
696
				state.setUnsuccessfull();
697
			}
698
		}
699
		throw new IllegalStateException("<writer> has no end tag");
700
	}
701

  
702
	private void registerFootnotes(MarkupImportState state, AnnotatableEntity entity, List<FootnoteDataHolder> footnotes) {
703
		for (FootnoteDataHolder footNote : footnotes) {
704
			registerFootnoteDemand(state, entity, footNote);
705
		}
706
	}
707

  
708
	private void registerGivenFootnote(MarkupImportState state, FootnoteDataHolder footnote) {
709
		state.registerFootnote(footnote);
710
		Set<AnnotatableEntity> demands = state.getFootnoteDemands(footnote.id);
711
		if (demands != null) {
712
			for (AnnotatableEntity entity : demands) {
713
				attachFootnote(state, entity, footnote);
714
			}
715
		}
716
	}
717

  
718
	private void registerGivenFigure(MarkupImportState state, XMLEvent next, String id, Media figure) {
719
		state.registerFigure(id, figure);
720
		Set<AnnotatableEntity> demands = state.getFigureDemands(id);
721
		if (demands != null) {
722
			for (AnnotatableEntity entity : demands) {
723
				attachFigure(state, next, entity, figure);
724
			}
725
		}
726
		save(figure, state);
727
	}
728

  
729
	private void registerFootnoteDemand(MarkupImportState state, AnnotatableEntity entity, FootnoteDataHolder footnote) {
730
		FootnoteDataHolder existingFootnote = state.getFootnote(footnote.ref);
731
		if (existingFootnote != null) {
732
			attachFootnote(state, entity, existingFootnote);
733
		} else {
734
			Set<AnnotatableEntity> demands = state.getFootnoteDemands(footnote.ref);
735
			if (demands == null) {
736
				demands = new HashSet<AnnotatableEntity>();
737
				state.putFootnoteDemands(footnote.ref, demands);
738
			}
739
			demands.add(entity);
740
		}
741
	}
742

  
743
	private void registerFigureDemand(MarkupImportState state, XMLEvent next, AnnotatableEntity entity, String figureRef) {
744
		Media existingFigure = state.getFigure(figureRef);
745
		if (existingFigure != null) {
746
			attachFigure(state, next, entity, existingFigure);
747
		} else {
748
			Set<AnnotatableEntity> demands = state.getFigureDemands(figureRef);
749
			if (demands == null) {
750
				demands = new HashSet<AnnotatableEntity>();
751
				state.putFigureDemands(figureRef, demands);
752
			}
753
			demands.add(entity);
754
		}
755
	}
756

  
757
	private void attachFootnote(MarkupImportState state, AnnotatableEntity entity, FootnoteDataHolder footnote) {
758
		AnnotationType annotationType = this.getAnnotationType(state, MarkupTransformer.uuidFootnote, "Footnote", "An e-flora footnote", "fn", null);
759
		Annotation annotation = Annotation.NewInstance(footnote.string, annotationType, getDefaultLanguage(state));
760
		// TODO transient objects
761
		entity.addAnnotation(annotation);
762
		save(entity, state);
763
	}
764

  
765
	private void attachFigure(MarkupImportState state, XMLEvent next, AnnotatableEntity entity, Media figure) {
766
		// IdentifiableEntity<?> toSave;
767
		if (entity.isInstanceOf(TextData.class)) {
768
			TextData deb = CdmBase.deproxy(entity, TextData.class);
769
			deb.addMedia(figure);
770
			// toSave = ((TaxonDescription)deb.getInDescription()).getTaxon();
771
		} else if (entity.isInstanceOf(SpecimenOrObservationBase.class)) {
772
			String message = "figures for specimen should be handled as Textdata";
773
			fireWarningEvent(message, next, 4);
774
			// toSave = ime;
775
		} else if (entity.isInstanceOf(IdentifiableMediaEntity.class)) {
776
			IdentifiableMediaEntity<?> ime = CdmBase.deproxy(entity, IdentifiableMediaEntity.class);
777
			ime.addMedia(figure);
778
			// toSave = ime;
779
		} else {
780
			String message = "Unsupported entity to attach media: %s";
781
			message = String.format(message, entity.getClass().getName());
782
			// toSave = null;
783
		}
784
		save(entity, state);
785
	}
786

  
787
	private Media handleFigure(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
788
		// FigureDataHolder result = new FigureDataHolder();
789

  
790
		Map<String, Attribute> attributes = getAttributes(parentEvent);
791
		String id = getAndRemoveAttributeValue(attributes, ID);
792
		String type = getAndRemoveAttributeValue(attributes, TYPE);
793
		String urlAttr = getAndRemoveAttributeValue(attributes, URL);
794
		checkNoAttributes(attributes, parentEvent);
795

  
796
		String urlString = null;
797
		String legendString = null;
798
		String titleString = null;
799
		String numString = null;
800
		String text = null;
801
		if (isNotBlank(urlAttr)){
802
			urlString = CdmUtils.Nz(state.getBaseMediaUrl()) + urlAttr;
803
		}
804
		while (reader.hasNext()) {
805
			XMLEvent next = readNoWhitespace(reader);
806
			if (isMyEndingElement(next, parentEvent)) {
807
				if (isNotBlank(text)){
808
					fireWarningEvent("Text not yet handled for figures: " + text, next, 4);
809
				}
810
				Media media = makeFigure(state, id, type, urlString, legendString, titleString, numString, next);
811
				return media;
812
			} else if (isStartingElement(next, FIGURE_LEGEND)) {
813
				// TODO same as figure string ?
814
				legendString = handleFootnoteString(state, reader, next);
815
			} else if (isStartingElement(next, FIGURE_TITLE)) {
816
				titleString = getCData(state, reader, next);
817
			} else if (isStartingElement(next, URL)) {
818
				String localUrl = getCData(state, reader, next);
819
				String url = CdmUtils.Nz(state.getBaseMediaUrl()) + localUrl;
820
				if (isBlank(urlString)){
821
					urlString = url;
822
				}
823
				if (! url.equals(urlString)){
824
					String message = "URL attribute and URL element differ. Attribute: %s, Element: %s";
825
					fireWarningEvent(String.format(message, urlString, url), next, 2);
826
				}
827
			} else if (isStartingElement(next, NUM)) {
828
				numString = getCData(state, reader, next);
829
			} else if (next.isCharacters()) {
830
				text += CdmUtils.concat("", text, next.asCharacters().getData());
831
			} else {
832
				fireUnexpectedEvent(next, 0);
833
			}
834
		}
835
		throw new IllegalStateException("<figure> has no end tag");
836
	}
837

  
838
	/**
839
	 * @param state
840
	 * @param id
841
	 * @param type
842
	 * @param urlString
843
	 * @param legendString
844
	 * @param titleString
845
	 * @param numString
846
	 * @param next
847
	 */
848
	private Media makeFigure(MarkupImportState state, String id, String type, String urlString, 
849
			String legendString, String titleString, String numString, XMLEvent next) {
850
		Media media = null;
851
		boolean isFigure = false;
852
		try {
853
			//TODO maybe everything is a figure as it is all taken from a book
854
			if ("lineart".equals(type)) {
855
				isFigure = true;
856
//				media = Figure.NewInstance(url.toURI(), null, null,	null);
857
			} else if (type == null || "photo".equals(type)
858
					|| "signature".equals(type)
859
					|| "others".equals(type)) {
860
				//TODO
861
			} else {
862
				String message = "Unknown figure type '%s'";
863
				message = String.format(message, type);
864
				fireWarningEvent(message, next, 2);
865
			}
866
			media = docImport.getImageMedia(urlString, docImport.getReadMediaData(), isFigure);
867
			
868
			if (media != null){
869
				// title
870
				if (StringUtils.isNotBlank(titleString)) {
871
					media.putTitle(getDefaultLanguage(state), titleString);
872
				}
873
				// legend
874
				if (StringUtils.isNotBlank(legendString)) {
875
					media.putDescription(getDefaultLanguage(state), legendString);
876
				}
877
				if (StringUtils.isNotBlank(numString)) {
878
					// TODO use concrete source (e.g. DAPHNIPHYLLACEAE in FM
879
					// vol.13)
880
					Reference<?> citation = state.getConfig().getSourceReference();
881
					media.addImportSource(  numString, "num", citation, null);
882
					// TODO name used in source if available
883
				}
884
				// TODO which citation
885
				if (StringUtils.isNotBlank(id)) {
886
					media.addImportSource(id, null, state.getConfig().getSourceReference(), null);
887
				} else {
888
					String message = "Figure id should never be empty or null";
889
					fireWarningEvent(message, next, 6);
890
				}
891

  
892
				// text
893
				// do nothing
894
				registerGivenFigure(state, next, id, media);
895
				
896
			}else{
897
				String message = "No media found: ";
898
				fireWarningEvent(message, next, 4);
899
			}
900
		} catch (MalformedURLException e) {
901
			String message = "Media uri has incorrect syntax: %s";
902
			message = String.format(message, urlString);
903
			fireWarningEvent(message, next, 4);
904
//		} catch (URISyntaxException e) {
905
//			String message = "Media uri has incorrect syntax: %s";
906
//			message = String.format(message, urlString);
907
//			fireWarningEvent(message, next, 4);
908
		}
909

  
910
		return media;
911
	}
912

  
913
	private FigureDataHolder handleFigureRef(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent)
914
			throws XMLStreamException {
915
		FigureDataHolder result = new FigureDataHolder();
916
		Map<String, Attribute> attributes = getAttributes(parentEvent);
917
		result.ref = getAndRemoveAttributeValue(attributes, REF);
918
		checkNoAttributes(attributes, parentEvent);
919

  
920
		// text is not handled, needed only for debugging purposes
921
		String text = "";
922
		while (reader.hasNext()) {
923
			XMLEvent next = readNoWhitespace(reader);
924
			if (isMyEndingElement(next, parentEvent)) {
925
				return result;
926
			} else if (isStartingElement(next, NUM)) {
927
				String num = getCData(state, reader, next);
928
				result.num = num; // num is not handled during import
929
			} else if (isStartingElement(next, FIGURE_PART)) {
930
				result.figurePart = getCData(state, reader, next);
931
			} else if (next.isCharacters()) {
932
				text += next.asCharacters().getData();
933
			} else {
934
				fireUnexpectedEvent(next, 0);
935
			}
936
		}
937
		throw new IllegalStateException("<figureRef> has no end tag");
938
	}
939

  
940
	private FootnoteDataHolder handleFootnote(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
941
		FootnoteDataHolder result = new FootnoteDataHolder();
942
		Map<String, Attribute> attributes = getAttributes(parentEvent);
943
		result.id = getAndRemoveAttributeValue(attributes, ID);
944
		// result.ref = getAndRemoveAttributeValue(attributes, REF);
945
		checkNoAttributes(attributes, parentEvent);
946

  
947
		while (reader.hasNext()) {
948
			XMLEvent next = readNoWhitespace(reader);
949
			if (isStartingElement(next, FOOTNOTE_STRING)) {
950
				String string = handleFootnoteString(state, reader, next);
951
				result.string = string;
952
			} else if (isMyEndingElement(next, parentEvent)) {
953
				return result;
954
			} else {
955
				fireUnexpectedEvent(next, 0);
956
			}
957
		}
958
		return result;
959
	}
960

  
961
	private FootnoteDataHolder handleFootnoteRef(MarkupImportState state,
962
			XMLEventReader reader, XMLEvent parentEvent)
963
			throws XMLStreamException {
964
		FootnoteDataHolder result = new FootnoteDataHolder();
965
		Map<String, Attribute> attributes = getAttributes(parentEvent);
966
		result.ref = getAndRemoveAttributeValue(attributes, REF);
967
		checkNoAttributes(attributes, parentEvent);
968

  
969
		// text is not handled, needed only for debugging purposes
970
		String text = "";
971
		while (reader.hasNext()) {
972
			XMLEvent next = readNoWhitespace(reader);
973
			// if (isStartingElement(next, FOOTNOTE_STRING)){
974
			// String string = handleFootnoteString(state, reader, next);
975
			// result.string = string;
976
			// }else
977
			if (isMyEndingElement(next, parentEvent)) {
978
				return result;
979
			} else if (next.isCharacters()) {
980
				text += next.asCharacters().getData();
981

  
982
			} else {
983
				fireUnexpectedEvent(next, 0);
984
			}
985
		}
986
		return result;
987
	}
988

  
989

  
990

  
991
	private String handleFootnoteString(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
992
		boolean isTextMode = true;
993
		String text = "";
994
		while (reader.hasNext()) {
995
			XMLEvent next = readNoWhitespace(reader);
996
			if (isMyEndingElement(next, parentEvent)) {
997
				return text;
998
			} else if (next.isEndElement()) {
999
				if (isEndingElement(next, FULL_NAME)) {
1000
					popUnimplemented(next.asEndElement());
1001
				} else if (isEndingElement(next, BR)) {
1002
					isTextMode = true;
1003
				} else if (isHtml(next)) {
1004
					text += getXmlTag(next);
1005
				} else {
1006
					handleUnexpectedEndElement(next.asEndElement());
1007
				}
1008
			} else if (next.isStartElement()) {
1009
				if (isStartingElement(next, FULL_NAME)) {
1010
					handleNotYetImplementedElement(next);
1011
				} else if (isStartingElement(next, GATHERING)) {
1012
					text += specimenImport.handleInLineGathering(state, reader, next);
1013
				} else if (isStartingElement(next, REFERENCES)) {
1014
					text += " " + handleInLineReferences(state, reader, next)+ " ";
1015
				} else if (isStartingElement(next, BR)) {
1016
					text += "<br/>";
1017
					isTextMode = false;
1018
				} else if (isStartingElement(next, NOMENCLATURE)) {
1019
					handleNotYetImplementedElement(next);
1020
				} else if (isHtml(next)) {
1021
					text += getXmlTag(next);
1022
				} else {
1023
					handleUnexpectedStartElement(next.asStartElement());
1024
				}
1025
			} else if (next.isCharacters()) {
1026
				if (!isTextMode) {
1027
					String message = "footnoteString is not in text mode";
1028
					fireWarningEvent(message, next, 6);
1029
				} else {
1030
					text += next.asCharacters().getData().trim(); 
1031
					// getCData(state, reader, next); does not work as we have inner tags like <references>
1032
				}
1033
			} else {
1034
				handleUnexpectedEndElement(next.asEndElement());
1035
			}
1036
		}
1037
		throw new IllegalStateException("<footnoteString> has no closing tag");
1038

  
1039
	}
1040

  
1041
	private String handleInLineReferences(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1042
		checkNoAttributes(parentEvent);
1043

  
1044
		boolean hasReference = false;
1045
		String text = "";
1046
		while (reader.hasNext()) {
1047
			XMLEvent next = readNoWhitespace(reader);
1048
			if (isMyEndingElement(next, parentEvent)) {
1049
				checkMandatoryElement(hasReference, parentEvent.asStartElement(), REFERENCE);
1050
				return text;
1051
			} else if (isStartingElement(next, REFERENCE)) {
1052
				text += handleInLineReference(state, reader, next);
1053
				hasReference = true;
1054
			} else {
1055
				handleUnexpectedElement(next);
1056
			}
1057
		}
1058
		throw new IllegalStateException("<References> has no closing tag");
1059
	}
1060

  
1061
	private String handleInLineReference(MarkupImportState state,XMLEventReader reader, XMLEvent parentEvent)throws XMLStreamException {
1062
		Reference<?> reference = nomenclatureImport.handleReference(state, reader, parentEvent);
1063
		String result = "<cdm:ref uuid='%s'>%s</ref>";
1064
		result = String.format(result, reference.getUuid(), reference.getTitleCache());
1065
		save(reference, state);
1066
		return result;
1067
	}
1068

  
1069
	private void handleFeature(MarkupImportState state, XMLEventReader reader, XMLEvent parentEvent) throws XMLStreamException {
1070
		Map<String, Attribute> attrs = getAttributes(parentEvent);
1071
		Boolean isFreetext = getAndRemoveBooleanAttributeValue(parentEvent, attrs, IS_FREETEXT, false);
1072
		String classValue =getAndRemoveRequiredAttributeValue(parentEvent, attrs, CLASS);
1073
		checkNoAttributes(attrs, parentEvent);
1074
		
1075
		
1076
		Feature feature = makeFeature(classValue, state, parentEvent, null);
1077
		Taxon taxon = state.getCurrentTaxon();
1078
		TaxonDescription taxonDescription = getTaxonDescription(taxon, state.getConfig().getSourceReference(), NO_IMAGE_GALLERY, CREATE_NEW);
1079
		// TextData figureHolderTextData = null; //for use with one TextData for
1080
		// all figure only
1081

  
1082
		boolean isDescription = feature.equals(Feature.DESCRIPTION());
1083
		DescriptionElementBase lastDescriptionElement = null;
1084
		
1085
		while (reader.hasNext()) {
1086
			XMLEvent next = readNoWhitespace(reader);
1087
			if (isMyEndingElement(next, parentEvent)) {
1088
				state.putFeatureToGeneralSorterList(feature);
1089
				return;
1090
			} else if (isEndingElement(next, DISTRIBUTION_LIST) || isEndingElement(next, HABITAT_LIST)) { 
1091
				// only handle list elements
1092
			} else if (isStartingElement(next, HEADING)) {
1093
				makeFeatureHeading(state, reader, classValue, feature, next);
1094
			} else if (isStartingElement(next, WRITER)) {
1095
				makeFeatureWriter(state, reader, feature, taxon, next);
1096
//			} else if (isStartingElement(next, DISTRIBUTION_LOCALITY)) {
1097
//				if (!feature.equals(Feature.DISTRIBUTION())) {
1098
//					String message = "Distribution locality only allowed for feature of type 'distribution'";
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff

Add picture from clipboard (Maximum size: 40 MB)