EDIT

// $Id$

/**

* Copyright (C) 2007 EDIT

* European Distributed Institute of Taxonomy 

* http://www.e-taxonomy.eu

* 

* The contents of this file are subject to the Mozilla Public License Version 1.1

* See LICENSE.TXT at the top of this package for the full license terms.

*/

package eu.etaxonomy.cdm.app.wp6.diptera;

import java.io.File;

import java.io.FileInputStream;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import org.apache.commons.lang.StringUtils;

import org.apache.log4j.Logger;

import org.springframework.transaction.TransactionStatus;

import au.com.bytecode.opencsv.CSVReader;

import eu.etaxonomy.cdm.api.application.CdmApplicationController;

import eu.etaxonomy.cdm.app.common.CdmDestinations;

import eu.etaxonomy.cdm.common.CdmUtils;

import eu.etaxonomy.cdm.database.DbSchemaValidation;

import eu.etaxonomy.cdm.database.ICdmDataSource;

import eu.etaxonomy.cdm.model.agent.Institution;

import eu.etaxonomy.cdm.model.occurrence.Collection;

import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;

import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;

/**

 * @author a.mueller

 * @date 07.04.2010

 *

 */

public class DipteraCollectionImport {

	private static final Logger logger = Logger.getLogger(DipteraCollectionImport.class);

	public static final File acronymsFile = new File("src/main/resources/collections/Acronyms.tab");

	//datasource for use from local main()

	static final ICdmDataSource cdmDestination = CdmDestinations.cdm_local_diptera();

	public boolean invoke(ICdmDataSource dataSource) {

		CdmApplicationController cdmApp = CdmApplicationController.NewInstance(dataSource, DbSchemaValidation.VALIDATE);

		//create collections

		TransactionStatus tx = cdmApp.startTransaction();

		Map<String, Collection> colletionMap = createCollections(cdmApp);

		//add collections to specimen

		addCollectionsToSpecimen(cdmApp, colletionMap);

		cdmApp.commitTransaction(tx);

		return true;

	}

	/**

	 * @param cdmApp

	 * @param colletionMap 

	 */

	private void addCollectionsToSpecimen(CdmApplicationController cdmApp, Map<String, Collection> colletionMap) {

		List<DerivedUnit> specimens = cdmApp.getOccurrenceService().list(DerivedUnit.class, null, null, null, null);

		for (SpecimenOrObservationBase<?> specOrObservBase : specimens){

			if (specOrObservBase.getRecordBasis().isPreservedSpecimen()){

				handleSingleSpecimen((DerivedUnit)specOrObservBase, colletionMap);

			}else{

				logger.warn("There are specimenOrObservationBase objects which are not of class Specimen. This is probably an error.");

			}

		}

		List<SpecimenOrObservationBase> specimenList = new ArrayList<SpecimenOrObservationBase>(specimens);

		cdmApp.getOccurrenceService().save(specimenList);

	}

	/**

	 * @param specimen 

	 * @param colletionMap

	 */

	private void handleSingleSpecimen(DerivedUnit specimen, Map<String, Collection> collectionMap) {

		String titleCache = specimen.getTitleCache();

		String collectionCode = getCollectionCode(titleCache);

		if (StringUtils.isBlank(collectionCode)){

			logger.warn("Collection code is empty for: " + titleCache);

		}else{

			Collection collection = collectionMap.get(collectionCode);

			if (collection != null){

				specimen.setCollection(collection);

			}else{

				logger.warn("Collection not found for code: " +  collectionCode + "; titleCache: " +  titleCache);

			}

		}

	}

	/**

	 * @param titleCache

	 * @return

	 */

	private String getCollectionCode(String titleCache) {

		String result = titleCache.trim();

		result = replaceBracket(result);

		result = replaceLastFullStop(result);

		result = replaceLastQuestionMark(result);

		result = parseLastUpperCase(result);

		return result;

	}

	/**

	 * @param result

	 * @return

	 */

	private String parseLastUpperCase(String string) {

		String result = "";

		String tmpString = string;

		int pos = tmpString.lastIndexOf(" ");

		if (pos>-1){

			tmpString = tmpString.substring(pos+1);

		}

		while (tmpString.length() > 0){

			int len = tmpString.length();

			char lastChar = tmpString.charAt(len-1);

			if (Character.isUpperCase( lastChar)){

				result = lastChar + result;

			}else{

				if (result.length() > 0){

					logger.warn("Collection code is not space separated: " + string);

				}

				break;

			}

			//remove last character

			tmpString = tmpString.substring(0, tmpString.length()-1);

		}

		return result;

	}

	/**

	 * @param result

	 * @return

	 */

	private String replaceLastQuestionMark(String string) {

		if (string.endsWith("?")){

			string = string.substring(0,string.length()-1).trim();

		}

		return string;

	}

	/**

	 * @param result

	 * @return

	 */

	private String replaceLastFullStop(String string) {

		if (string.endsWith(".")){

			string = string.substring(0,string.length()-1).trim();

		}

		return string;

	}

	/**

	 * @param result

	 * @return

	 */

	private String replaceBracket(String string) {

		if (string.endsWith("]")){

			int pos  = string.indexOf("[");

			if (pos >0){

				string = string.substring(0, pos).trim();

			}else{

				logger.warn("Closing bracket has no opening bracket in: " + string);

			}

		}

		return string;

	}

	/**

	 * @param cdmApp

	 */

	private Map<String, Collection> createCollections(CdmApplicationController cdmApp) {

		Map<String, Collection> collectionMap = new HashMap<String, Collection>(); 

		List<String[]> lines = getLines();

		for (String[] line:lines){

			Collection collection = makeLine(line);

			collectionMap.put(collection.getCode(), collection);

		}

		cdmApp.getCollectionService().save(collectionMap.values());

//			for (Collection collection: collectionMap.values()){

//				System.out.println(collection.getTitleCache());

//			}

		return collectionMap;

	}

	private Collection makeLine(String[] line) {

		String code = line[0];

		String instituteName = line[1];

		String lowerInstitutionName = line[2];

		String higherInstitutionName = line[3];

		String location = line[4];

		String country = line[5];

		//create objects

		Collection collection = Collection.NewInstance();

		collection.setCode(code);

		Institution institution = Institution.NewInstance();

		institution.setCode(code);

		institution.setName(instituteName);

		if (StringUtils.isNotBlank(lowerInstitutionName)){

			Institution lowerInstitution = Institution.NewInstance();

			lowerInstitution.setName(lowerInstitutionName);

			lowerInstitution.setIsPartOf(institution);

		}

		if (StringUtils.isNotBlank(higherInstitutionName)){

			Institution higherInstitution = Institution.NewInstance();

			higherInstitution.setName(higherInstitutionName);

			institution.setIsPartOf(higherInstitution);

		}

		collection.setInstitute(institution);

		String locationAndCountry = CdmUtils.concat("/", location, country);

		collection.setTownOrLocation(locationAndCountry);

		String titleCache = CdmUtils.concat(", ", new String[]{instituteName, lowerInstitutionName, higherInstitutionName, location, country});

		collection.setTitleCache(titleCache, true);

		return collection;

	}

	private List<String[]> getLines() {

		List<String[]> result = new ArrayList<String[]>();

		try {

			InputStream inStream = new FileInputStream(acronymsFile);

			InputStreamReader inputStreamReader = new InputStreamReader(inStream, "UTF8");

			CSVReader reader = new CSVReader(inputStreamReader, '\t');

			String [] nextLine = reader.readNext();

			while ((nextLine = reader.readNext()) != null) {

				if (nextLine.length == 0){

					continue;

				}

				result.add(nextLine);

			}

			return result;

		} catch (Exception e) {

			logger.error(e + " " + e.getCause() + " " + e.getMessage());

			for(StackTraceElement ste : e.getStackTrace()) {

				logger.error(ste);

			}

			throw new RuntimeException(e);

		}

	}

	/**

	 * @param args

	 */

	public static void main(String[] args) {

		try {

			DipteraCollectionImport collectionImport = new DipteraCollectionImport();

			collectionImport.invoke(cdmDestination);

//			String titleCache = "Peru. Mouth of Rio Pachitea. ST 2R SMT. [fig. of male abdomen]";

//			String collectionCode = collectionImport.getCollectionCode(titleCache);

//			System.out.println(collectionCode);

		} catch (Exception e) {

			e.printStackTrace();

			System.exit(-1);

		}

	}

}

/**

* Copyright (C) 2007 EDIT

* European Distributed Institute of Taxonomy 

* http://www.e-taxonomy.eu

* 

* The contents of this file are subject to the Mozilla Public License Version 1.1

* See LICENSE.TXT at the top of this package for the full license terms.

*/

/**

* Copyright (C) 2007 EDIT

* European Distributed Institute of Taxonomy 

* http://www.e-taxonomy.eu

* 

* The contents of this file are subject to the Mozilla Public License Version 1.1

* See LICENSE.TXT at the top of this package for the full license terms.

*/

package eu.etaxonomy.cdm.app.wp6.diptera;

import java.util.ArrayList;

import java.util.HashSet;

import java.util.List;

import java.util.Set;

import java.util.regex.Pattern;

import org.apache.log4j.Logger;

import org.springframework.transaction.TransactionStatus;

import eu.etaxonomy.cdm.api.application.CdmApplicationController;

import eu.etaxonomy.cdm.api.application.ICdmApplicationConfiguration;

import eu.etaxonomy.cdm.app.common.CdmDestinations;

import eu.etaxonomy.cdm.database.DbSchemaValidation;

import eu.etaxonomy.cdm.database.ICdmDataSource;

import eu.etaxonomy.cdm.io.common.TdwgAreaProvider;

import eu.etaxonomy.cdm.model.common.Language;

import eu.etaxonomy.cdm.model.description.DescriptionBase;

import eu.etaxonomy.cdm.model.description.DescriptionElementBase;

import eu.etaxonomy.cdm.model.description.Distribution;

import eu.etaxonomy.cdm.model.description.Feature;

import eu.etaxonomy.cdm.model.description.PresenceAbsenceTerm;

import eu.etaxonomy.cdm.model.description.TaxonDescription;

import eu.etaxonomy.cdm.model.description.TextData;

import eu.etaxonomy.cdm.model.location.NamedArea;

import eu.etaxonomy.cdm.model.taxon.Taxon;

import eu.etaxonomy.cdm.model.taxon.TaxonBase;

/**

 * @author a.mueller

 * @created 17.10.2008

 * @version 1.0

 */

public class DipteraDistributionParser {

	private static final Logger logger = Logger.getLogger(DipteraDistributionParser.class);

	private static ICdmDataSource cdmDestination = CdmDestinations.cdm_local_diptera();

	final static String epiSplitter = "(\\s+|\\[|\\]|\\(|\\))"; //( ' '+| '(' | ')'| '[' | ']' )

	static Pattern pattern = null;

	protected void doDistribution(ICdmApplicationConfiguration app){

		pattern = Pattern.compile(epiSplitter); 

	    TransactionStatus txStatus = app.startTransaction();

		List<TaxonBase> taxa = app.getTaxonService().list(null, null, null, null, null);

		for (TaxonBase taxon: taxa ){

			if (taxon instanceof Taxon){

		//		unlazyDescription(app, (Taxon)taxon);

				Set<TaxonDescription> descriptions = ((Taxon) taxon).getDescriptions();

				for (DescriptionBase description: descriptions){

					Set<DescriptionElementBase> descElements = new HashSet<DescriptionElementBase>();

					descElements.addAll(description.getElements());

					for (DescriptionElementBase descEl: descElements){

						if (descEl.getFeature().equals(Feature.OCCURRENCE())){

							if (descEl instanceof TextData){

								String occString = ((TextData)descEl).getText(Language.ENGLISH());

								parseOccurenceString(occString, description);

								//app.getTaxonService().saveTaxon(taxon);

							}

						}

					}

				}

			}

		}

		System.out.println("Unknowns: ");

		for (String unknown: unrekognizedStrings){

			System.out.println(unknown);

		}

		System.out.println("Distributions not recognized: " + countNot);

		System.out.println("Distributions created: " + countYes);

		app.commitTransaction(txStatus);

	}

	static Set<String> unrekognizedStrings = new HashSet<String>();

	static int countNot = 0;

	static int countYes = 0;

	private void parseOccurenceString(String occString, DescriptionBase desc){

		System.out.println(occString);

		if (occString != null){

			String[] words = pattern.split(occString);

			int i = 0;

			int countSkip = 0;

			for (String word: words){

				if (word.contains("U.S.A")){

					logger.warn("U.S.A.");

				}

				boolean isDoubtful = false;

				if (countSkip > 0){

					countSkip--;

				}else if(word.trim().length() == 0){

					//skip

				}else{

					if (word.endsWith(":") && word.length()<=4){

						//Higher area

						//TODO

					}else{

						word = word.trim();

						if (word.contains("?")){

							isDoubtful = true;

							word = word.replace("?", "");

						}

						word = adaptWordsToTdwg(word);

						if (! "".equals(word) && ! TdwgAreaProvider.isTdwgAreaLabel(word) && ! TdwgAreaProvider.isTdwgAreaAbbreviation(word) && ! isDoubleArea(word)){

							for (countSkip = 1; countSkip <= 6; countSkip++){

								word = word.trim();

								if (! TdwgAreaProvider.isTdwgAreaLabel(word) && ! TdwgAreaProvider.isTdwgAreaAbbreviation(word) && ! isDoubleArea(word)){

									if (words.length > i + countSkip){

										word = word + " " + words[i + countSkip];

									}

									if (word.contains("?")){

										isDoubtful = true;

										word = word.replace("?", "");

									}

									word = adaptWordsToTdwg(word);

									if ("".equals(word)){

										break;

									}

								}else{

									break;

								}

							}

						}

						if ("".equals(word)){

							//countSkip = countSkip;

						}else if (! TdwgAreaProvider.isTdwgAreaLabel(word)  && ! TdwgAreaProvider.isTdwgAreaAbbreviation(word) &&  ! isDoubleArea(word)  ){

							if (word.contains("?")){

								logger.warn("XXX");

							}

							countNot++;

							System.out.println("   False:" + countNot + ": " + word);

							unrekognizedStrings.add(word);

							countSkip = 0;

						}else{

							if (word.equals("Netherlands")){

								if ( countSkip < 0 && words[i + 1].startsWith("Antilles")){

									word = "Netherlands Antilles";

									countSkip=2;

								}

							}

							PresenceAbsenceTerm term = PresenceAbsenceTerm.PRESENT();

							if (isDoubleArea(word)){

								NamedArea[] doubleArea = getDoubleArea(word);

								for (NamedArea area : doubleArea){

									Distribution distr = Distribution.NewInstance(area, term);

									desc.addElement(distr);

								}

							}else{

								NamedArea area;

								if (TdwgAreaProvider.isTdwgAreaLabel(word)){

									area = TdwgAreaProvider.getAreaByTdwgLabel(word);

								}else{

									area = TdwgAreaProvider.getAreaByTdwgAbbreviation(word);

								}

								if (isDoubtful){

									term = PresenceAbsenceTerm.INTRODUCED_PRESENCE_QUESTIONABLE();

								}

								Distribution distr = Distribution.NewInstance(area, term);

								desc.addElement(distr);

							}

							countYes++;

							System.out.println("      True:" + countYes + ": " + word);

							countSkip--;

						}

					}

				}

				i++;

			}

		}

	}

	private boolean isDoubleArea(String word){

		if ("Canary and Madeira Is.".equalsIgnoreCase(word) || 

				"southern Europe".equalsIgnoreCase(word) ||

				"former USSR: North and Central European territory".equalsIgnoreCase(word)

				){

			return true;

		}else{

			return false;

		}

	}

	private NamedArea[] getDoubleArea(String word){

		NamedArea[] result = new NamedArea[2];

		if ("Canary and Madeira Is.".equalsIgnoreCase(word)){

			 result[0] = TdwgAreaProvider.getAreaByTdwgAbbreviation("CNY");

			 result[1] = TdwgAreaProvider.getAreaByTdwgAbbreviation("MDR");

		}else if ("southern Europe".equalsIgnoreCase(word)){

			 result[0] = TdwgAreaProvider.getAreaByTdwgAbbreviation("12");

			 result[1] = TdwgAreaProvider.getAreaByTdwgAbbreviation("13");

		}else if ("former USSR: North and Central European territory".equalsIgnoreCase(word)){

			 result[0] = TdwgAreaProvider.getAreaByTdwgAbbreviation("RUN-OO");

			 result[1] = TdwgAreaProvider.getAreaByTdwgAbbreviation("RUC-OO");

		}else{

			logger.warn("Double area not recognized");

		}

		return result;

	}

	static List<String> stopWords = new ArrayList<String>();

	static List<String> unknownAreas = new ArrayList<String>();

	static List<String> higherAreas = new ArrayList<String>();

	private String adaptWordsToTdwg(String word){

		word = word.replace(",", "").replace(";", "");

		if (! word.contains("U.S.A")){

			word = word.replace(",", "").replace(".", "").replace(";", "");

		}else{

			word = word.replace(",", "").replace(";", "");

		}

		word = word.trim();

		if (word.endsWith("Is")){

			word = word + ".";

		}

		if (stopWords.size() == 0){

			initStopWords();

		}

		word = word.replace("Russia [North European territory]", "North European Russia");

		word = word.replace("Russia North European territory", "North European Russia");

		word = word.replace("Russia: North European territory", "North European Russia");

		word = word.replace("Russia: North European territory", "North European Russia");

		word = word.replace("Amber", "amber");

		word = word.replace("Prince Edward Is.", "Marion-Prince Edward Is.");

		//or word = word.replace("Prince Edward Is.", "Prince Edward I.");

		word = word.replace("Bahama Is.", "Bahamas");

		word = word.replace("Comores Is.", "Comoros");

		word = word.replace("former Yugoslavia", "Yugoslavia");

		word = word.replace("former Czechoslovakia", "Czechoslovakia");

		word = word.replace("Rhodesia", "Zimbabwe");

		word = word.replace("The Gambia", "Gambia, The");

		if (!word.contains("El Salvador")){

			word = word.replace("Salvador", "El Salvador");	

		}

		word = word.replace("Vera Cruz", "Veracruz");

		word = word.replace("Turkmenia", "Turkmenistan");

		word = word.replace("Qu\u00E9beck", "Qu\u00E9bec");

		word = word.replace("Quebeck", "Qu\u00E9bec");

		word = word.replace("Quebec", "Qu\u00E9bec");

		if (!word.contains("Gambia, The")){

			word = word.replace("Gambia", "Gambia, The");

		}

		word = word.replace("Mariana Is.", "Marianas");

		word = word.replace("Kenia", "Kenya");

		word = word.replace("Central Africa", "Central African Republic");

		word = word.replace("Canal Zone", "");

		//word = word.replace("Panama", "Panamá");

		word = word.replace("Panama", "Panam\u00E1");

		if (! word.contains("New South Wales")){

			word = word.replace("Wales", "Great Britain");

		}

		word = word.replace("Java", "Jawa");

		word = word.replace("former USSR: North European territory", "North European Russia");

		word = word.replace("former USSR: South European territory", "South European Russia");

		word = word.replace("former USSR: Soviet Middle Asia", "Middle Asia");

		word = word.replace("St Kitts-Nevis", "St.Kitts-Nevis");

		word = word.replace("oceanian islands", "Pacific");

		word = word.replace("Ussuri region", "Primorye");

		word = word.replace("Galapagos Is.", "Gal\u00E1pagos");

		word = word.replace("Tarapac\u00E1", "Tarapaca");

		word = word.replace("Reunion", "R\u00E9union");

		if (! word.contains("Is.")){

			word = word.replace("Galapagos", "Gal\u00E1pagos");

		}

		//word = word.replace("Galapagos Is.", "Galápagos");

		if (! word.contains("Peninsular")){

			word = word.replace("Malaysia", "Peninsular Malaysia");

		}

		word = word.replace("Polynesic Is.", "South Solomons");

		word = word.replace("Usbek SSR", "Uzbekistan");

		word = word.replace("Mexican amber", "Mexico");

		word = word.replace("Marocco", "Morocco");

		if (! word.contains("Tobago")){

			word = word.replace("Trinidad", "Trinidad-Tobago");

		}

		if (! word.contains("Trinidad")){

			word = word.replace("Tobago", "Trinidad-Tobago");

		}

		word = word.replace("Haiti", "Haiti");  

		word = word.replace("Moluccas", "Maluku");

		word = word.replace("Belau", "Palau");

		word = word.replace("Dominican amber", "Dominican Republic");

		if (! word.contains("Russian")){

			word = word.replace("Far East", "Russian Far East");

		}

		word = word.replace("Tahiti", "Society Is.");

		word = word.replace("Iraque", "Iraq");

		word = word.replace("Wake Island", "Wake I.");

		if (! word.contains("I.")){

			word = word.replace("Johnston I", "Johnston I.");

			word = word.replace("Wake I", "Wake I.");

			word = word.replace("Clipperton I", "Clipperton I.");

		}

		if (! word.contains("Provinces")){

			word = word.replace("Cape Province", "Cape Provinces");

		}

		word = word.replace("Eastern Cape Provinces", "Eastern Cape Province");

		word = word.replace("Western Cape Provinces", "Western Cape Province");

		if (! word.contains("Barbuda")){

			word = word.replace("Antigua", "Antigua-Barbuda");

		}

		if (! word.contains("St.")){

			word = word.replace("St Vincent", "St.Vincent");

			word = word.replace("St Lucia", "St.Lucia");

			word = word.replace("St Helena", "St.Helena");

		}

		word = word.replace("Asia-tropical", "Asia-Tropical");

		word = word.replace("Society Islands", "Society Is.");

		word = word.replace("Virgin Islands", "Virgin Is.");

		word = word.replace("Canary Islands", "Canary Is.");

		word = word.replace("Rhode Island", "Rhode I.");

		word = word.replace("Rodriguez", "Rodrigues");

		word = word.replace("British Colombia", "British Columbia");

		word = word.replace("Bermudas", "Bermuda");

		word = word.replace("Tunesia", "Tunisia");

		word = word.replace("Santos S\u00E3o Paulo", "S\u00E3o Paulo");

		word = word.replace("Transvaal", "Northern Provinces");

		word = word.replace("Tucum\u00E1n", "Tucuman");

//		if (!word.contains("Netherlands")){

//			

//		}

//		unknownAreas.add("Baltic amber");  

//		unknownAreas.add("Arabia"); 

		for (String stopWord : stopWords){

			if (stopWord.equals(word)){

				System.out.println("         STOP: " + word);

				return "";

			}

		}

		for (String unknownArea : unknownAreas){

			if (unknownArea.equals(word)){

				System.out.println("         UNKNOWN: " + word);

				return "";

			}

		}

		for (String higherArea : higherAreas){

			if (higherArea.equals(word)){

				return "";

			}

		}

		//higher regions

		return word;

	}

	private void initStopWords(){

		stopWords.add("and");

		stopWords.add("Is");

		stopWords.add("Is.");

		stopWords.add("Islands");

		stopWords.add("Island");

		stopWords.add("of");

		stopWords.add("areas");

		stopWords.add("USA");

		stopWords.add("Australia"); //except for Australia only

		stopWords.add("Argentina");		

		//unknownAreas.add("Panama");

		unknownAreas.add("South Africa");

		unknownAreas.add("Chile");

		unknownAreas.add("Baltic amber");  

		unknownAreas.add("Arabia"); 

		higherAreas.add("AF");

		higherAreas.add("OR");

		higherAreas.add("PA");

		higherAreas.add("AU");

		higherAreas.add("NE");

		higherAreas.add("NT");

	}

	/**

	 * @param args

	 */

	public static void main(String[] args) {

		CdmApplicationController app = null;

		DbSchemaValidation val = DbSchemaValidation.UPDATE;

		app = CdmApplicationController.NewInstance(cdmDestination, val);

		DipteraDistributionParser dipDist = new DipteraDistributionParser();

		if (app != null){

			dipDist.doDistribution(app);

		}else{

			logger.warn("No Application Context");

		}

	}

}