cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelCommonNamesImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.berlinModel.in;
  11
  12 import java.sql.ResultSet;
  13 import java.sql.SQLException;
  14 import java.util.ArrayList;
  15 import java.util.HashMap;
  16 import java.util.HashSet;
  17 import java.util.List;
  18 import java.util.Map;
  19 import java.util.Set;
  20 import java.util.SortedSet;
  21 import java.util.TreeSet;
  22 import java.util.UUID;
  23
  24 import org.apache.commons.lang.StringUtils;
  25 import org.apache.log4j.Logger;
  26 import org.springframework.stereotype.Component;
  27
  28 import eu.etaxonomy.cdm.common.CdmUtils;
  29 import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelCommonNamesImportValidator;
  30 import eu.etaxonomy.cdm.io.common.IOValidator;
  31 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  32 import eu.etaxonomy.cdm.io.common.Source;
  33 import eu.etaxonomy.cdm.model.common.Annotation;
  34 import eu.etaxonomy.cdm.model.common.AnnotationType;
  35 import eu.etaxonomy.cdm.model.common.CdmBase;
  36 import eu.etaxonomy.cdm.model.common.DescriptionElementSource;
  37 import eu.etaxonomy.cdm.model.common.Extension;
  38 import eu.etaxonomy.cdm.model.common.ExtensionType;
  39 import eu.etaxonomy.cdm.model.common.Language;
  40 import eu.etaxonomy.cdm.model.common.Representation;
  41 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  42 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  43 import eu.etaxonomy.cdm.model.location.NamedArea;
  44 import eu.etaxonomy.cdm.model.location.TdwgArea;
  45 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  46 import eu.etaxonomy.cdm.model.reference.Reference;
  47 import eu.etaxonomy.cdm.model.taxon.Taxon;
  48 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  49
  50 /**
  51  *
  52  * @author a.mueller
  53  * @created 20.03.2008
  54  * @version 1.0
  55  */
  56 @Component
  57 public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
  58         private static final Logger logger = Logger.getLogger(BerlinModelCommonNamesImport.class);
  59
  60         public static final UUID REFERENCE_LANGUAGE_ISO639_2_UUID = UUID.fromString("40c4f8dd-3d9c-44a4-b77a-76e137a89a5f");
  61         public static final UUID REFERENCE_LANGUAGE_STRING_UUID = UUID.fromString("2a1b678f-c27d-48c1-b43e-98fd0d426305");
  62         public static final UUID STATUS_ANNOTATION_UUID = UUID.fromString("e3f7b80a-1286-458d-812c-5e818f731968");
  63
  64         public static final String NAMESPACE = "common name";
  65
  66
  67         private static final String pluralString = "common names";
  68         private static final String dbTableName = "emCommonName";
  69
  70
  71         //map that stores the regions (named areas) and makes them accessible via the regionFk
  72         private Map<String, NamedArea> regionMap = new HashMap<String, NamedArea>();
  73
  74
  75
  76         public BerlinModelCommonNamesImport(){
  77                 super();
  78         }
  79
  80
  81
  82         @Override
  83         protected String getIdQuery(BerlinModelImportState state) {
  84                 String result = " SELECT CommonNameId FROM emCommonName ";
  85                 if (StringUtils.isNotBlank(state.getConfig().getCommonNameFilter())){
  86                         result += " AND " + state.getConfig().getCommonNameFilter();
  87                 }
  88
  89                 return result;
  90         }
  91
  92
  93
  94         /* (non-Javadoc)
  95          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
  96          */
  97         @Override
  98         protected String getRecordQuery(BerlinModelImportConfigurator config) {
  99                 String recordQuery = "";
 100                 recordQuery =
 101                         " SELECT emCommonName.CommonNameId, emCommonName.CommonName, PTaxon.RIdentifier AS taxonId, emCommonName.PTNameFk, emCommonName.RefFk AS refId, emCommonName.Status, " +
 102                                 " emCommonName.RegionFks, emCommonName.MisNameRefFk, emCommonName.NameInSourceFk , emCommonName.Created_When, emCommonName.Updated_When, emCommonName.Created_Who, emCommonName.Updated_Who, emCommonName.Note as Notes," +
 103                         " regionLanguage.Language AS regionLanguage, languageCommonName.Language, languageCommonName.LanguageOriginal, languageCommonName.ISO639_1, languageCommonName.ISO639_2, " +
 104                         " emLanguageRegion.Region, emLanguageReference.RefFk as languageRefRefFk, emLanguageReference.ReferenceShort, " +
 105                         " emLanguageReference.ReferenceLong, emLanguageReference.LanguageFk, languageReferenceLanguage.Language AS refLanguage, " +
 106                         " languageReferenceLanguage.ISO639_2 AS refLanguageIso639_2, regionLanguage.ISO639_2 AS regionLanguageIso, " +
 107                         " misappliedTaxon.RIdentifier AS misappliedTaxonId " +
 108                 " FROM emLanguage as regionLanguage RIGHT OUTER JOIN " +
 109                         " emLanguageRegion ON regionLanguage.LanguageId = emLanguageRegion.LanguageFk RIGHT OUTER JOIN " +
 110                         " emLanguage AS languageReferenceLanguage RIGHT OUTER JOIN " +
 111                         " emLanguageReference ON languageReferenceLanguage.LanguageId = emLanguageReference.LanguageFk RIGHT OUTER JOIN " +
 112                         " emCommonName INNER JOIN " +
 113                         " PTaxon ON emCommonName.PTNameFk = PTaxon.PTNameFk AND emCommonName.PTRefFk = PTaxon.PTRefFk ON " +
 114                         " emLanguageReference.ReferenceId = emCommonName.LanguageRefFk LEFT OUTER JOIN " +
 115                         " emLanguage AS languageCommonName ON emCommonName.LanguageFk = languageCommonName.LanguageId ON " +
 116                         " emLanguageRegion.RegionId = emCommonName.RegionFks LEFT OUTER JOIN " +
 117                         " PTaxon as misappliedTaxon ON emCommonName.PTNameFk = misappliedTaxon.PTNameFk AND emCommonName.MisNameRefFk = misappliedTaxon.PTRefFk " +
 118                         " WHERE emCommonName.CommonNameId IN (" + ID_LIST_TOKEN + ")";
 119                 return recordQuery;
 120         }
 121
 122
 123
 124         @Override
 125         protected void doInvoke(BerlinModelImportState state) {
 126                 try {
 127                         makeRegions(state);
 128                 } catch (Exception e) {
 129                         logger.error("Error when creating common name regions:" + e.getMessage());
 130                         state.setUnsuccessfull();
 131                 }
 132                 super.doInvoke(state);
 133                 return;
 134         }
 135
 136         /**
 137          * @param state
 138          *
 139          */
 140         private void makeRegions(BerlinModelImportState state) {
 141                 try {
 142                         SortedSet<Integer> regionFks = new TreeSet<Integer>();
 143                         Source source = state.getConfig().getSource();
 144
 145                         getRegionFks(state, regionFks, source);
 146                         //concat filter string
 147                         String sqlWhere = getSqlWhere(regionFks);
 148
 149                         //get E+M - TDWG Mapping
 150                         Map<String, String> emTdwgMap = getEmTdwgMap(source);
 151                         //fill regionMap
 152                         fillRegionMap(source, sqlWhere, emTdwgMap);
 153
 154                         return;
 155                 } catch (NumberFormatException e) {
 156                         e.printStackTrace();
 157                         state.setUnsuccessfull();
 158                         return;
 159                 } catch (SQLException e) {
 160                         e.printStackTrace();
 161                         state.setUnsuccessfull();
 162                         return;
 163                 }
 164         }
 165
 166
 167         /* (non-Javadoc)
 168          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)
 169          */
 170         public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state)  {
 171                 boolean success = true ;
 172                 BerlinModelImportConfigurator config = state.getConfig();
 173                 Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();
 174                 Map<String, Taxon> taxonMap = (Map<String, Taxon>) partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);
 175                 Map<String, TaxonNameBase> taxonNameMap = (Map<String, TaxonNameBase>) partitioner.getObjectMap(BerlinModelTaxonNameImport.NAMESPACE);
 176
 177                 Map<String, Reference> biblioRefMap = (Map<String, Reference>) partitioner.getObjectMap(BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE);
 178                 Map<String, Reference> nomRefMap = (Map<String, Reference>) partitioner.getObjectMap(BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE);
 179
 180                 Map<String, Language> iso6392Map = new HashMap<String, Language>();
 181
 182         //      logger.warn("MisappliedNameRefFk  not yet implemented for Common Names");
 183
 184                 ResultSet rs = partitioner.getResultSet();
 185                 try{
 186                         while (rs.next()){
 187
 188                                 //create TaxonName element
 189                                 Object commonNameId = rs.getObject("CommonNameId");
 190                                 int taxonId = rs.getInt("taxonId");
 191                                 Object refId = rs.getObject("refId");
 192                                 Object ptNameFk = rs.getObject("PTNameFk");
 193                                 String commonNameString = rs.getString("CommonName");
 194                                 String iso639_2 = rs.getString("ISO639_2");
 195                                 String iso639_1 = rs.getString("ISO639_1");
 196                                 String languageString = rs.getString("Language");
 197                                 String originalLanguageString = rs.getString("LanguageOriginal");
 198                                 Object misNameRefFk = rs.getObject("MisNameRefFk");
 199                                 Object languageRefRefFk = rs.getObject("languageRefRefFk");
 200                                 String refLanguage = rs.getString("refLanguage");
 201                                 String refLanguageIso639_2 = rs.getString("refLanguageIso639_2");
 202                                 String status = rs.getString("Status");
 203                                 Object nameInSourceFk = rs.getObject("NameInSourceFk");
 204                                 Object misappliedTaxonId = rs.getObject("misappliedTaxonId");
 205
 206                                 //regions
 207                                 String region = rs.getString("Region");
 208                                 String regionFks  = rs.getString("RegionFks");
 209                                 String[] regionFkSplit = regionFks.split(",");
 210
 211                                 //commonNameString
 212                                 if (CdmUtils.isEmpty(commonNameString)){
 213                                         String message = "CommonName is empty or null. Do not import record for taxon " + taxonId;
 214                                         logger.warn(message);
 215                                         continue;
 216                                 }
 217
 218                                 //taxon
 219                                 Taxon taxon = null;
 220                                 TaxonBase taxonBase = null;
 221                                 taxonBase  = taxonMap.get(String.valueOf(taxonId));
 222                                 if (taxonBase == null){
 223                                         logger.warn("Taxon (" + taxonId + ") could not be found. Common name " + commonNameString + " not imported");
 224                                         continue;
 225                                 }else if (! taxonBase.isInstanceOf(Taxon.class)){
 226                                         logger.warn("taxon (" + taxonId + ") is not accepted. Can't import common name " +  commonNameId);
 227                                         continue;
 228                                 }else{
 229                                         taxon = CdmBase.deproxy(taxonBase, Taxon.class);
 230                                 }
 231
 232                                 //Language
 233                                 Language language = getAndHandleLanguage(iso6392Map, iso639_2, iso639_1, languageString, originalLanguageString);
 234
 235                                 //CommonTaxonName
 236                                 List<CommonTaxonName> commonTaxonNames = new ArrayList<CommonTaxonName>();
 237                                 for (String regionFk : regionFkSplit){ //
 238                                         CommonTaxonName commonTaxonName;
 239                                         if (commonTaxonNames.size() == 0){
 240                                                 commonTaxonName = CommonTaxonName.NewInstance(commonNameString, language);
 241                                         }else{
 242                                                 commonTaxonName = (CommonTaxonName)commonTaxonNames.get(0).clone();
 243                                         }
 244                                         commonTaxonNames.add(commonTaxonName);
 245                                         regionFk = regionFk.trim();
 246                                         NamedArea area = regionMap.get(regionFk);
 247                                         if (area == null){
 248                                                 if (regionFkSplit.length > 1 && StringUtils.isNotBlank(regionFk)){
 249                                                         logger.warn("Area for " + regionFk + " not defined.");
 250                                                 }else{
 251                                                         //no region is defined
 252                                                 }
 253                                         }else{
 254                                                 commonTaxonName.setArea(area);
 255                                                 TaxonDescription description = getDescription(taxon);
 256                                                 description.addElement(commonTaxonName);
 257                                         }
 258                                 }
 259
 260                                 //Reference/Source
 261                                 String strRefId = String.valueOf(refId);
 262                                 String languageRefFk = String.valueOf(languageRefRefFk);
 263                                 if (! CdmUtils.nullSafeEqual(strRefId, languageRefFk)){
 264                                         //use strRefId if languageRefFk is null
 265                                         if (languageRefRefFk == null){
 266                                                 languageRefFk = strRefId;
 267                                         }else{
 268                                                 logger.warn("CommonName.RefFk (" + CdmUtils.Nz(strRefId) + ") and LanguageReference.RefFk " + CdmUtils.Nz(languageRefFk) + " are not equal. I will import only languageReference.RefFk");
 269                                         }
 270                                 }
 271
 272
 273                                 Reference reference = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, String.valueOf(languageRefRefFk));
 274                                 String microCitation = null;
 275                                 String originalNameString = null;
 276
 277                                 TaxonNameBase nameUsedInSource = taxonNameMap.get(String.valueOf(nameInSourceFk));
 278                                 if (nameInSourceFk != null && nameUsedInSource == null){
 279                                         logger.warn("Name used in source (" + nameInSourceFk + ") was not found");
 280                                 }
 281                                 DescriptionElementSource source = DescriptionElementSource.NewInstance(reference, microCitation, nameUsedInSource, originalNameString);
 282                                 for (CommonTaxonName commonTaxonName : commonTaxonNames){
 283                                         commonTaxonName.addSource(source);
 284                                 }
 285
 286                                 //MisNameRef
 287                                 if (misNameRefFk != null){
 288                                         //Taxon misappliedName = getMisappliedName(biblioRefMap, nomRefMap, misNameRefFk, taxon);
 289                                         Taxon misappliedName = null;
 290                                         if (misappliedTaxonId != null){
 291                                                 misappliedName = taxonMap.get(String.valueOf(misappliedTaxonId));
 292                                         }else{
 293                                                 TaxonNameBase taxonName = taxonNameMap.get(String.valueOf(ptNameFk));
 294                                                 Reference sec = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, String.valueOf(misNameRefFk));
 295                                                 if (taxonName == null || sec == null){
 296                                                         logger.info("Taxon name or misapplied name reference is null for common name " + commonNameId);
 297                                                 }else{
 298                                                         misappliedName = Taxon.NewInstance(taxonName, sec);
 299                                                         taxaToSave.add(misappliedName);
 300                                                 }
 301                                         }
 302                                         if (misappliedName != null){
 303                                                 taxon.addMisappliedName(misappliedName, config.getSourceReference(), null);
 304                                                 TaxonDescription misappliedNameDescription = getDescription(misappliedName);
 305                                                 for (CommonTaxonName commonTaxonName : commonTaxonNames){
 306                                                         CommonTaxonName commonNameClone = (CommonTaxonName)commonTaxonName.clone();
 307                                                         misappliedNameDescription.addElement(commonNameClone);
 308                                                 }
 309                                         }else{
 310                                                 logger.warn("Misapplied name is null for common name " + commonNameId);
 311                                         }
 312
 313                                 }
 314
 315
 316                                 //reference extensions
 317                                 if (reference != null){
 318                                         if (CdmUtils.isNotEmpty(refLanguage)){
 319                                                 ExtensionType refLanguageExtensionType = getExtensionType( state, REFERENCE_LANGUAGE_STRING_UUID, "reference language","The language of the reference","ref. lang.");
 320                                                 Extension.NewInstance(reference, refLanguage, refLanguageExtensionType);
 321                                         }
 322
 323                                         if (CdmUtils.isNotEmpty(refLanguageIso639_2)){
 324                                                 ExtensionType refLanguageIsoExtensionType = getExtensionType( state, REFERENCE_LANGUAGE_ISO639_2_UUID, "reference language iso 639-2","The iso 639-2 code of the references language","ref. lang. 639-2");
 325                                                 Extension.NewInstance(reference, refLanguageIso639_2, refLanguageIsoExtensionType);
 326                                         }
 327                                 }else if (CdmUtils.isNotEmpty(refLanguage) || CdmUtils.isNotEmpty(refLanguageIso639_2)){
 328                                         logger.warn("Reference is null (" + languageRefRefFk + ") but refLanguage (" + CdmUtils.Nz(refLanguage) + ") or iso639_2 (" + CdmUtils.Nz(refLanguageIso639_2) + ") was not null");
 329                                 }
 330
 331                                 //status
 332                                 if (CdmUtils.isNotEmpty(status)){
 333                                         AnnotationType statusAnnotationType = getAnnotationType( state, STATUS_ANNOTATION_UUID, "status","The status of this object","status", null);
 334                                         Annotation annotation = Annotation.NewInstance(status, statusAnnotationType, Language.DEFAULT());
 335                                         for (CommonTaxonName commonTaxonName : commonTaxonNames){
 336                                                 commonTaxonName.addAnnotation(annotation);
 337                                         }
 338
 339                                 }
 340
 341                                 //Notes
 342                                 for (CommonTaxonName commonTaxonName : commonTaxonNames){
 343                                         doIdCreatedUpdatedNotes(state, commonTaxonName, rs, String.valueOf(commonNameId), NAMESPACE);
 344                                 }
 345                                 partitioner.startDoSave();
 346                                 taxaToSave.add(taxon);
 347
 348                         }
 349                 } catch (SQLException e) {
 350                         logger.error("SQLException:" +  e);
 351                         return false;
 352                 } catch (ClassCastException e) {
 353                         e.printStackTrace();
 354                 }
 355
 356                 //      logger.info( i + " names handled");
 357                 getTaxonService().save(taxaToSave);
 358                 return success;
 359
 360         }
 361
 362
 363
 364         /**
 365          * Not used anymore. Use MisappliedName RIdentifier instead
 366          * @param biblioRefMap
 367          * @param nomRefMap
 368          * @param misNameRefFk
 369          * @param taxon
 370          */
 371         private boolean isFirstMisappliedName = true;
 372         private Taxon getMisappliedName(Map<String, Reference> biblioRefMap, Map<String, Reference> nomRefMap, Object misNameRefFk, Taxon taxon) {
 373                 Taxon misappliedTaxon = null;
 374                 Reference<?> misNameRef = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, String.valueOf(misNameRefFk));
 375                 misappliedTaxon = Taxon.NewInstance(taxon.getName(), misNameRef);
 376                 Set<String> includeProperty = new HashSet<String>();
 377                 try {
 378 //                      //IMatchStrategy matchStrategy = DefaultMatchStrategy.NewInstance(TaxonBase.class);
 379 //                      //List<TaxonBase> misappliedList1 = getCommonService().findMatching(misappliedTaxon, matchStrategy);
 380                         List<TaxonBase> misappliedList = getTaxonService().list(misappliedTaxon, includeProperty, null, null, null, null);
 381                         if (misappliedList.size() > 0){
 382                                 misappliedTaxon = CdmBase.deproxy(misappliedList.get(0), Taxon.class);
 383                         }
 384                 } catch (ClassCastException e) {
 385                         logger.error(e.getMessage());
 386                         if (isFirstMisappliedName){
 387                                 e.printStackTrace();
 388                                 isFirstMisappliedName = false;
 389                         }
 390                 }
 391                 return misappliedTaxon;
 392         }
 393
 394
 395
 396         /**
 397          * @param iso6392Map
 398          * @param iso639_2
 399          * @param languageString
 400          * @param originalLanguageString
 401          * @return
 402          */
 403         private Language getAndHandleLanguage(Map<String, Language> iso639Map,  String iso639_2, String iso639_1, String languageString, String originalLanguageString) {
 404                 Language language;
 405                 if (CdmUtils.isNotEmpty(iso639_2)|| CdmUtils.isNotEmpty(iso639_1)  ){
 406                         //TODO test performance, implement in state
 407                         language = getLanguageFromIsoMap(iso639Map, iso639_2, iso639_1);
 408
 409                         if (language == null){
 410                                 language = getTermService().getLanguageByIso(iso639_2);
 411                                 iso639Map.put(iso639_2, language);
 412                                 if (language == null){
 413                                         language = getTermService().getLanguageByIso(iso639_1);
 414                                         iso639Map.put(iso639_1, language);
 415                                 }
 416                                 if (language == null){
 417                                         logger.warn("Language for code ISO693-2 '" + iso639_2 + "' and ISO693-1 '" + iso639_1 + "' was not found");
 418                                 }
 419                         }
 420                 } else if ("unknown".equals(languageString)){
 421                         language = Language.UNKNOWN_LANGUAGE();
 422                 }else{
 423                         logger.warn("language ISO 639_1 and ISO 639_2 were empty for " + languageString);
 424                         language = null;
 425                 }
 426                 addOriginalLanguage(language, originalLanguageString);
 427                 return language;
 428         }
 429
 430
 431         /**
 432          * @param iso639Map
 433          * @param iso639_2
 434          * @param iso639_1
 435          * @return
 436          */
 437         private Language getLanguageFromIsoMap(Map<String, Language> iso639Map, String iso639_2, String iso639_1) {
 438                 Language language;
 439                 language = iso639Map.get(iso639_2);
 440                 if (language == null){
 441                         language = iso639Map.get(iso639_1);
 442                 }
 443                 return language;
 444         }
 445
 446         /**
 447          * @param language
 448          * @param originalLanguageString
 449          */
 450         private void addOriginalLanguage(Language language,     String originalLanguageString) {
 451                 if (CdmUtils.isEmpty(originalLanguageString)){
 452                         return;
 453                 }else if (language == null){
 454                         logger.warn("Language could not be defined, but originalLanguageString exists: " + originalLanguageString);
 455                 }else {
 456                         Representation representation = language.getRepresentation(language);
 457                         if (representation == null){
 458                                 language.addRepresentation(Representation.NewInstance(originalLanguageString, originalLanguageString, originalLanguageString, language));
 459                                 getTermService().saveOrUpdate(language);
 460                         }
 461                 }
 462
 463         }
 464
 465
 466
 467         /**
 468          * @param state
 469          * @param regionFks
 470          * @param source
 471          * @return
 472          * @throws SQLException
 473          */
 474         private void getRegionFks(BerlinModelImportState state, SortedSet<Integer> regionFks,
 475                         Source source) throws SQLException {
 476                 String sql = " SELECT DISTINCT RegionFks FROM emCommonName";
 477                 ResultSet rs = source.getResultSet(sql);
 478                 while (rs.next()){
 479                         String strRegionFks = rs.getString("RegionFks");
 480                         String[] regionFkArray = strRegionFks.split(",");
 481                         for (String regionFk: regionFkArray){
 482                                 regionFk = regionFk.trim();
 483                                 if (! StringUtils.isNumeric(regionFk) || "".equals(regionFk)  ){
 484                                         state.setUnsuccessfull();
 485                                         logger.warn("RegionFk is not numeric: " + regionFk);
 486                                 }else{
 487                                         regionFks.add(Integer.valueOf(regionFk));
 488                                 }
 489                         }
 490                 }
 491                 return;
 492         }
 493
 494
 495
 496         /**
 497          * @param source
 498          * @param sqlWhere
 499          * @param emTdwgMap
 500          * @throws SQLException
 501          */
 502         private void fillRegionMap(Source source, String sqlWhere,
 503                         Map<String, String> emTdwgMap) throws SQLException {
 504                 String sql;
 505                 ResultSet rs;
 506                 sql = " SELECT RegionId, Region FROM emLanguageRegion WHERE RegionId IN ("+ sqlWhere+ ") ";
 507                 rs = source.getResultSet(sql);
 508                 while (rs.next()){
 509                         Object regionId = rs.getObject("RegionId");
 510                         String region = rs.getString("Region");
 511                         String[] splitRegion = region.split("-");
 512                         if (splitRegion.length <= 1){
 513                                 NamedArea newArea = NamedArea.NewInstance(region, region, null);
 514                                 getTermService().save(newArea);
 515                                 regionMap.put(String.valueOf(regionId), newArea);
 516                                 logger.warn("Found new area: " +  region);
 517                         }else if (splitRegion.length == 2){
 518                                 String emCode = splitRegion[1].trim();
 519                                 String tdwgCode = emTdwgMap.get(emCode);
 520                                 if (StringUtils.isNotBlank(tdwgCode) ){
 521                                         NamedArea tdwgArea = getNamedArea(tdwgCode);
 522                                         regionMap.put(String.valueOf(regionId), tdwgArea);
 523                                 }else{
 524                                         logger.warn("emCode did not map to valid tdwgCode: " +  CdmUtils.Nz(emCode) + "->" + CdmUtils.Nz(tdwgCode));
 525                                 }
 526                         }
 527                 }
 528         }
 529
 530
 531         /**
 532          * @param tdwgCode
 533          */
 534         private NamedArea getNamedArea(String tdwgCode) {
 535                 NamedArea area;
 536                 if (tdwgCode.equalsIgnoreCase("Ab")){
 537                         area = NamedArea.NewInstance("Azerbaijan (including Nakhichevan)", "Azerbaijan & Nakhichevan", "Ab");
 538                         getTermService().save(area);
 539                 }else if (tdwgCode.equalsIgnoreCase("Rf")){
 540                         area = NamedArea.NewInstance("The Russian Federation", "The Russian Federation", "Rf");
 541                         getTermService().save(area);
 542                 }else if (tdwgCode.equalsIgnoreCase("Uk")){
 543                         area = NamedArea.NewInstance("Ukraine (including Crimea)", "Ukraine & Crimea", "Uk");
 544                         getTermService().save(area);
 545                 }else{
 546                         area = TdwgArea.getAreaByTdwgAbbreviation(tdwgCode);
 547                 }
 548                 return area;
 549         }
 550
 551
 552
 553         /**
 554          * @param regionFks
 555          * @return
 556          */
 557         private String getSqlWhere(SortedSet<Integer> regionFks) {
 558                 String sqlWhere = "";
 559                 for (Integer regionFk : regionFks){
 560                         sqlWhere += regionFk + ",";
 561                 }
 562                 sqlWhere = sqlWhere.substring(0, sqlWhere.length()-1);
 563                 return sqlWhere;
 564         }
 565
 566
 567
 568         /**
 569          * @param source
 570          * @throws SQLException
 571          */
 572         private Map<String, String> getEmTdwgMap(Source source) throws SQLException {
 573                 String sql;
 574                 ResultSet rs;
 575                 Map<String, String> emTdwgMap = new HashMap<String, String>();
 576                 sql = " SELECT EmCode, TDWGCode FROM emArea ";
 577                 rs = source.getResultSet(sql);
 578                 while (rs.next()){
 579                         String emCode = rs.getString("EMCode");
 580                         String TDWGCode = rs.getString("TDWGCode");
 581                         if (StringUtils.isNotBlank(emCode) ){
 582                                 emCode = emCode.trim();
 583                                 if (emCode.equalsIgnoreCase("Ab") || emCode.equalsIgnoreCase("Rf")|| emCode.equalsIgnoreCase("Uk") ){
 584                                         emTdwgMap.put(emCode, emCode);
 585                                 }else if (StringUtils.isNotBlank(TDWGCode)){
 586                                         emTdwgMap.put(emCode, TDWGCode.trim());
 587                                 }
 588                         }
 589                 }
 590                 return emTdwgMap;
 591         }
 592
 593
 594
 595
 596         /**
 597          * Returns the first non-image gallery description. Creates a new one if no description exists.
 598          * @param taxon
 599          * @return
 600          */
 601         private TaxonDescription getDescription(Taxon taxon) {
 602                 TaxonDescription result = null;
 603                 for (TaxonDescription taxonDescription : taxon.getDescriptions()){
 604                         if (! taxonDescription.isImageGallery()){
 605                                 result = taxonDescription;
 606                         }
 607                 }
 608                 if (result == null){
 609                         result = TaxonDescription.NewInstance(taxon);
 610                 }
 611                 return result;
 612         }
 613
 614         /* (non-Javadoc)
 615          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
 616          */
 617         public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {
 618                 String nameSpace;
 619                 Class<?> cdmClass;
 620                 Set<String> idSet;
 621                 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
 622
 623                 try{
 624                         Set<String> taxonIdSet = new HashSet<String>();
 625                         Set<String> nameIdSet = new HashSet<String>();
 626                         Set<String> referenceIdSet = new HashSet<String>();
 627                         while (rs.next()){
 628                                 handleForeignKey(rs, taxonIdSet, "taxonId");
 629                                 handleForeignKey(rs, taxonIdSet, "misappliedTaxonId");
 630                                 handleForeignKey(rs, referenceIdSet, "refId");
 631                                 handleForeignKey(rs, referenceIdSet, "languageRefRefFk");
 632                                 handleForeignKey(rs, nameIdSet, "NameInSourceFk");
 633                                 handleForeignKey(rs, nameIdSet, "PTNameFk");
 634                                 handleForeignKey(rs, referenceIdSet, "MisNameRefFk");
 635                         }
 636
 637                         //name map
 638                         nameSpace = BerlinModelTaxonNameImport.NAMESPACE;
 639                         cdmClass = TaxonNameBase.class;
 640                         idSet = nameIdSet;
 641                         Map<String, TaxonNameBase<?,?>> nameMap = (Map<String, TaxonNameBase<?,?>>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 642                         result.put(nameSpace, nameMap);
 643
 644                         //name map
 645                         nameSpace = BerlinModelTaxonImport.NAMESPACE;
 646                         cdmClass = Taxon.class;
 647                         idSet = taxonIdSet;
 648                         Map<String, TaxonNameBase<?,?>> taxonMap = (Map<String, TaxonNameBase<?,?>>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 649                         result.put(nameSpace, taxonMap);
 650
 651                         //nom reference map
 652                         nameSpace = BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE;
 653                         cdmClass = Reference.class;
 654                         idSet = referenceIdSet;
 655                         Map<String, Reference> nomReferenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 656                         result.put(nameSpace, nomReferenceMap);
 657
 658                         //biblio reference map
 659                         nameSpace = BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE;
 660                         cdmClass = Reference.class;
 661                         idSet = referenceIdSet;
 662                         Map<String, Reference<?>> biblioReferenceMap = (Map<String, Reference<?>>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 663                         result.put(nameSpace, biblioReferenceMap);
 664
 665                 } catch (SQLException e) {
 666                         throw new RuntimeException(e);
 667                 } catch (NullPointerException nep){
 668                         logger.error("NullPointerException in getRelatedObjectsForPartition()");
 669                 }
 670                 return result;
 671         }
 672
 673
 674
 675         /* (non-Javadoc)
 676          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
 677          */
 678         @Override
 679         protected boolean doCheck(BerlinModelImportState state){
 680                 IOValidator<BerlinModelImportState> validator = new BerlinModelCommonNamesImportValidator();
 681                 return validator.validate(state);
 682         }
 683
 684
 685         /* (non-Javadoc)
 686          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getTableName()
 687          */
 688         @Override
 689         protected String getTableName() {
 690                 return dbTableName;
 691         }
 692
 693         /* (non-Javadoc)
 694          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getPluralString()
 695          */
 696         @Override
 697         public String getPluralString() {
 698                 return pluralString;
 699         }
 700
 701         /* (non-Javadoc)
 702          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
 703          */
 704         protected boolean isIgnore(BerlinModelImportState state){
 705                 return ! state.getConfig().isDoCommonNames();
 706         }
 707
 708 }