cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/excel/common/ExcelTaxonOrSpecimenImportBase.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.excel.common;
  11
  12 import java.util.HashMap;
  13 import java.util.List;
  14 import java.util.Set;
  15 import java.util.UUID;
  16
  17 import org.apache.commons.lang.StringUtils;
  18 import org.apache.log4j.Logger;
  19
  20 import eu.etaxonomy.cdm.api.service.pager.Pager;
  21 import eu.etaxonomy.cdm.common.CdmUtils;
  22 import eu.etaxonomy.cdm.io.excel.common.ExcelRowBase.PostfixTerm;
  23 import eu.etaxonomy.cdm.io.specimen.excel.in.SpecimenCdmExcelImportState;
  24 import eu.etaxonomy.cdm.io.specimen.excel.in.SpecimenRow;
  25 import eu.etaxonomy.cdm.model.common.CdmBase;
  26 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  27 import eu.etaxonomy.cdm.model.common.Extension;
  28 import eu.etaxonomy.cdm.model.common.ExtensionType;
  29 import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
  30 import eu.etaxonomy.cdm.model.description.Feature;
  31
  32 /**
  33  * @author a.mueller
  34  * @date 12.07.2011
  35  */
  36 public abstract class ExcelTaxonOrSpecimenImportBase<STATE extends ExcelImportState<? extends ExcelImportConfiguratorBase, ROW>, ROW extends ExcelRowBase> extends ExcelImporterBase<STATE> {
  37         private static final Logger logger = Logger.getLogger(ExcelTaxonOrSpecimenImportBase.class);
  38
  39
  40         protected static final String CDM_UUID_COLUMN = "(?i)(CdmUuid)";
  41         protected static final String IGNORE_COLUMN = "(?i)(Ignore|Not)";
  42
  43
  44         protected static final String RANK_COLUMN = "(?i)(Rank)";
  45         protected static final String FULL_NAME_COLUMN = "(?i)(FullName)";
  46         protected static final String TAXON_UUID_COLUMN = "(?i)(taxonUuid)";
  47         protected static final String FAMILY_COLUMN = "(?i)(Family)";
  48         protected static final String GENUS_COLUMN = "(?i)(Genus)";
  49         protected static final String SPECIFIC_EPITHET_COLUMN = "(?i)(SpecificEpi(thet)?)";
  50         protected static final String INFRASPECIFIC_EPITHET_COLUMN = "(?i)(InfraSpecificEpi(thet)?)";
  51
  52         @Override
  53         protected void analyzeRecord(HashMap<String, String> record, STATE state) {
  54                 Set<String> keys = record.keySet();
  55
  56         ROW row = createDataHolderRow();
  57         state.setCurrentRow(row);
  58
  59         for (String originalKey: keys) {
  60                 KeyValue keyValue = makeKeyValue(record, originalKey, state);
  61                 if (StringUtils.isBlank(keyValue.value)){
  62                         continue;
  63                 }
  64                 if (isBaseColumn(keyValue)){
  65                         handleBaseColumn(keyValue, row);
  66                 }else{
  67                         analyzeSingleValue(keyValue, state);
  68                 }
  69         }
  70         return;
  71         }
  72
  73         protected abstract ROW createDataHolderRow();
  74
  75         /**
  76          * Analyzes a single record value and fills the row instance accordingly.
  77          * @param keyValue
  78          * @param state
  79          * @return
  80          */
  81         protected abstract void analyzeSingleValue(KeyValue keyValue, STATE state);
  82
  83         /**
  84          *      DataHolder class for all key and value information for a cell.
  85          * Value is the value of the cell (as String). Key is the main attribute, further defined by postfix,
  86          * and in case of multiple values indexed.
  87          * TODO doc for refXXX
  88          */
  89         protected class KeyValue{
  90                 public KeyValue() {}
  91
  92                 //original Key
  93                 public String originalKey;
  94                 //value
  95                 public String value;
  96                 //atomized key
  97                 public String key;
  98                 public String postfix;
  99                 public int index = 0;
 100                 public SourceType refType;
 101                 public int refIndex = 0;
 102                 public boolean hasError = false;
 103                 public boolean isKeyData() {
 104                         return (refType == null);
 105                 }
 106                 public boolean isLanguage(){
 107                         return (refType.isLanguage());
 108                 }
 109         }
 110
 111         public enum SourceType{
 112                 Author("RefAuthor"),
 113                 Title("RefTitle"),
 114                 Year("RefYear"),
 115                 RefExtension("RefExt(ension)?"),
 116                 Language("Lang") //strictly not a reference, so some refactoring/renaming is needed
 117                 ;
 118
 119                 String keyMatch = null;
 120                 private SourceType(String keyName){
 121                         this.keyMatch = keyName;
 122                 }
 123
 124
 125                 boolean isLanguage(){
 126                         return (this.equals(Language));
 127                 }
 128
 129                 static SourceType byKeyName(String str){
 130                         if (StringUtils.isBlank(str)){
 131                                 return null;
 132                         }
 133                         for (SourceType type : SourceType.values()){
 134                                 if (str.matches("(?i)(" + type.keyMatch + ")")){
 135                                         return type;
 136                                 }
 137                         }
 138                         return null;
 139                 }
 140
 141                 static boolean isKeyName(String str){
 142                         return (byKeyName(str) != null);
 143                 }
 144
 145         }
 146
 147
 148         /**
 149          * @param record
 150          * @param originalKey
 151          * @param state
 152          * @param keyValue
 153          * @return
 154          */
 155         protected KeyValue makeKeyValue(HashMap<String, String> record, String originalKey, STATE state) {
 156                 KeyValue keyValue = new KeyValue();
 157                 keyValue.originalKey = originalKey;
 158                 String indexedKey = CdmUtils.removeDuplicateWhitespace(originalKey.trim()).toString();
 159                 String[] split = indexedKey.split("_");
 160                 int current = 0;
 161                 //key
 162                 keyValue.key = split[current++];
 163                 //postfix
 164                 if (split.length > current && ! isRefType(split[current]) && ! isInteger(split[current]) ){
 165                         keyValue.postfix = split[current++];
 166                 }
 167                 //index
 168                 if (split.length > current && isInteger(split[current]) ){
 169                         keyValue.index = Integer.valueOf(split[current++]);
 170                 }else{
 171                         keyValue.index = 0;
 172                 }
 173                 //source
 174                 if (split.length > current){
 175                         //refType
 176                         if (isRefType(split[current])){
 177                                 String refTypeStr = split[current++];
 178                                 keyValue.refType = SourceType.byKeyName(refTypeStr);
 179                                 if (keyValue.refType == null){
 180                                         String message = "Unmatched source key: " + refTypeStr;
 181                                         fireWarningEvent(message, state, 10);
 182                                         logger.warn(message);
 183                                 }
 184                         }else {
 185                                 String message = "RefType expected at %d position of key. But %s is no valid reftype";
 186                                 message = String.format(message, current, split[current]);
 187                                 fireWarningEvent(message, state, 10);
 188                                 logger.warn(message);
 189                                 keyValue.hasError  = true;
 190                         }
 191                         //ref index
 192                         if (split.length > current){
 193                                  if (isInteger(split[current])){
 194                                          keyValue.refIndex = Integer.valueOf(split[current++]);
 195                                  }else{
 196                                         String message = "Ref index expected at position %d of key. But %s is no valid reftype";
 197                                         message = String.format(message, current, split[current]);
 198                                         fireWarningEvent(message, state, 10);
 199                                         logger.warn(message);
 200                                         keyValue.hasError = true;
 201                                  }
 202                         }else {
 203                                 keyValue.refIndex = 0;
 204                         }
 205
 206                 }
 207                 if (split.length > current){
 208                         String message = "Key has unexpected part at position %d of key. %s (and following parts) can not be handled";
 209                         message = String.format(message, current, split[current]);
 210                         fireWarningEvent(message, state, 10);
 211                         logger.warn(message);
 212                         keyValue.hasError = true;
 213                 }
 214
 215                 //TODO shouldn't we use originalKey here??
 216                 String value = (String) record.get(indexedKey);
 217                 if (! StringUtils.isBlank(value)) {
 218                         if (logger.isDebugEnabled()) { logger.debug(keyValue.key + ": " + value); }
 219                         value = CdmUtils.removeDuplicateWhitespace(value.trim()).toString();
 220                         keyValue.value = value;
 221                 }else{
 222                         keyValue.value = null;
 223                 }
 224                 return keyValue;
 225         }
 226
 227
 228         private boolean isRefType(String string) {
 229                 return SourceType.isKeyName(string);
 230         }
 231
 232
 233         private boolean handleBaseColumn(KeyValue keyValue, ExcelRowBase row) {
 234                 String key = keyValue.key;
 235                 String value = keyValue.value;
 236                 if (key.matches(CDM_UUID_COLUMN)) {
 237                         row.setCdmUuid(UUID.fromString(value)); //VALIDATE UUID
 238                 }
 239                 return true;
 240         }
 241
 242         private boolean isBaseColumn(KeyValue keyValue) {
 243                 String key = keyValue.key;
 244                 if (key.matches(CDM_UUID_COLUMN)){
 245                         return true;
 246                 } else if(keyValue.key.matches(IGNORE_COLUMN)) {
 247                         logger.debug("Ignored column" + keyValue.originalKey);
 248                         return true;
 249                 }
 250                 return false;
 251         }
 252
 253         protected boolean isInteger(String value){
 254                 try {
 255                         Integer.valueOf(value);
 256                         return true;
 257                 } catch (NumberFormatException e) {
 258                         return false;
 259                 }
 260         }
 261
 262
 263         protected boolean analyzeFeatures(STATE state, KeyValue keyValue) {
 264                 String key = keyValue.key;
 265                 Pager<DefinedTermBase> features = getTermService().findByTitle(Feature.class, key, null, null, null, null, null, null);
 266
 267                 // FOR DEBUGGING ONLY
 268                 System.out.println("XXX ANALYSE feature: " + key);
 269                 if (key != null && key.equals("Description")){
 270                         Feature desc = (Feature)getTermService().find(UUID.fromString("9087cdcd-8b08-4082-a1de-34c9ba9fb493"));
 271                         if (desc != null){
 272                                 desc.setProtectedTitleCache(true);
 273                                 System.out.println("XXXXXXXXXXXXX Description feature title cache is: " + desc.getTitleCache());
 274                         }else{
 275                                 System.out.println("XXXXXXXXXXXXXXX No description feature found.");
 276                         }
 277                 }
 278                 //************************
 279
 280                 if (features.getCount() > 1){
 281                         String message = "More than one feature found matching key " + key;
 282                         fireWarningEvent(message, state, 4);
 283                         return false;
 284                 }else if (features.getCount() == 0){
 285                         return false;
 286                 }else{
 287                         Feature feature = CdmBase.deproxy(features.getRecords().get(0), Feature.class);
 288                         ROW row = state.getCurrentRow();
 289                         if ( keyValue.isKeyData()){
 290                                 row.putFeature(feature.getUuid(), keyValue.index, keyValue.value);
 291                         }else if (keyValue.isLanguage()){
 292                                 row.putFeatureLanguage(feature.getUuid(), keyValue.index, keyValue.value);
 293                         }else{
 294                                 row.putFeatureSource(feature.getUuid(), keyValue.index, keyValue.refType, keyValue.value, keyValue.refIndex);
 295                         }
 296                         return true;
 297                 }
 298         }
 299
 300
 301         protected void handleExtensions(IdentifiableEntity<?> identifiable, SpecimenRow row, SpecimenCdmExcelImportState state) {
 302                 List<PostfixTerm> extensions = row.getExtensions();
 303
 304                 for (PostfixTerm exType : extensions){
 305                         ExtensionType extensionType = state.getPostfixExtensionType(exType.postfix);
 306
 307                         Extension extension = Extension.NewInstance();
 308                         extension.setType(extensionType);
 309                         extension.setValue(exType.term);
 310                         identifiable.addExtension(extension);
 311                 }
 312
 313         }
 314
 315
 316         protected void fireWarningEvent(String message, STATE state, int severity) {
 317                 fireWarningEvent(message, "Record" + state.getCurrentLine(), severity, 1);
 318         }
 319 }