Source code

001package org.hl7.fhir.r4.context;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033
034import java.io.File;
035import java.io.FileNotFoundException;
036import java.io.FileOutputStream;
037import java.io.IOException;
038import java.io.OutputStreamWriter;
039import java.util.ArrayList;
040import java.util.HashMap;
041import java.util.List;
042import java.util.Map;
043
044import org.apache.commons.lang3.StringUtils;
045import org.hl7.fhir.exceptions.FHIRException;
046import org.hl7.fhir.r4.context.IWorkerContext.ValidationResult;
047import org.hl7.fhir.r4.formats.IParser.OutputStyle;
048import org.hl7.fhir.r4.formats.JsonParser;
049import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent;
050import org.hl7.fhir.r4.model.CodeableConcept;
051import org.hl7.fhir.r4.model.Coding;
052import org.hl7.fhir.r4.model.UriType;
053import org.hl7.fhir.r4.model.ValueSet;
054import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent;
055import org.hl7.fhir.r4.model.ValueSet.ConceptSetFilterComponent;
056import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent;
057import org.hl7.fhir.r4.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
058import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
059import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
060import org.hl7.fhir.utilities.TextFile;
061import org.hl7.fhir.utilities.Utilities;
062import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
063import org.hl7.fhir.utilities.validation.ValidationOptions;
064
065import com.google.gson.JsonElement;
066import com.google.gson.JsonNull;
067import com.google.gson.JsonObject;
068import com.google.gson.JsonPrimitive;
069
070/**
071 * This implements a two level cache. 
072 *  - a temporary cache for remmbering previous local operations
073 *  - a persistent cache for rembering tx server operations
074 *  
075 * the cache is a series of pairs: a map, and a list. the map is the loaded cache, the list is the persiistent cache, carefully maintained in order for version control consistency
076 * 
077 * @author graha
078 *
079 */
080public class TerminologyCache {
081  public static final boolean TRANSIENT = false;
082  public static final boolean PERMANENT = true;
083  private static final String NAME_FOR_NO_SYSTEM = "all-systems";
084  private static final String ENTRY_MARKER = "-------------------------------------------------------------------------------------";
085  private static final String BREAK = "####";
086
087  private SystemNameKeyGenerator systemNameKeyGenerator = new SystemNameKeyGenerator();
088
089  protected SystemNameKeyGenerator getSystemNameKeyGenerator() {
090    return systemNameKeyGenerator;
091  }
092
093  public class SystemNameKeyGenerator {
094    public static final String SNOMED_SCT_CODESYSTEM_URL = "http://snomed.info/sct";
095    public static final String RXNORM_CODESYSTEM_URL = "http://www.nlm.nih.gov/research/umls/rxnorm";
096    public static final String LOINC_CODESYSTEM_URL = "http://loinc.org";
097    public static final String UCUM_CODESYSTEM_URL = "http://unitsofmeasure.org";
098
099    public static final String HL7_TERMINOLOGY_CODESYSTEM_BASE_URL = "http://terminology.hl7.org/CodeSystem/";
100    public static final String HL7_SID_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/sid/";
101    public static final String HL7_FHIR_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/";
102
103    public static final String ISO_CODESYSTEM_URN = "urn:iso:std:iso:";
104    public static final String LANG_CODESYSTEM_URN = "urn:ietf:bcp:47";
105    public static final String MIMETYPES_CODESYSTEM_URN = "urn:ietf:bcp:13";
106
107    public static final String _11073_CODESYSTEM_URN = "urn:iso:std:iso:11073:10101";
108    public static final String DICOM_CODESYSTEM_URL = "http://dicom.nema.org/resources/ontology/DCM";
109
110    public String getNameForSystem(String system) {
111      final int lastPipe = system.lastIndexOf('|');
112      final String systemBaseName = lastPipe == -1 ? system : system.substring(0,lastPipe);
113      final String systemVersion = lastPipe == -1 ? null : system.substring(lastPipe + 1);
114
115      if (systemBaseName.equals(SNOMED_SCT_CODESYSTEM_URL))
116        return getVersionedSystem("snomed", systemVersion);
117      if (systemBaseName.equals(RXNORM_CODESYSTEM_URL))
118        return getVersionedSystem("rxnorm", systemVersion);
119      if (systemBaseName.equals(LOINC_CODESYSTEM_URL))
120        return getVersionedSystem("loinc", systemVersion);
121      if (systemBaseName.equals(UCUM_CODESYSTEM_URL))
122        return getVersionedSystem("ucum", systemVersion);
123      if (systemBaseName.startsWith(HL7_SID_CODESYSTEM_BASE_URL))
124        return getVersionedSystem(normalizeBaseURL(HL7_SID_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
125      if (systemBaseName.equals(_11073_CODESYSTEM_URN))
126        return getVersionedSystem("11073", systemVersion);
127      if (systemBaseName.startsWith(ISO_CODESYSTEM_URN))
128        return getVersionedSystem("iso"+systemBaseName.substring(ISO_CODESYSTEM_URN.length()).replace(":", ""), systemVersion);
129      if (systemBaseName.startsWith(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL))
130        return getVersionedSystem(normalizeBaseURL(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
131      if (systemBaseName.startsWith(HL7_FHIR_CODESYSTEM_BASE_URL))
132        return getVersionedSystem(normalizeBaseURL(HL7_FHIR_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
133      if (systemBaseName.equals(LANG_CODESYSTEM_URN))
134        return getVersionedSystem("lang", systemVersion);
135      if (systemBaseName.equals(MIMETYPES_CODESYSTEM_URN))
136        return getVersionedSystem("mimetypes", systemVersion);
137      if (systemBaseName.equals(DICOM_CODESYSTEM_URL))
138        return getVersionedSystem("dicom", systemVersion);
139      return getVersionedSystem(systemBaseName.replace("/", "_").replace(":", "_").replace("?", "X").replace("#", "X"), systemVersion);
140    }
141
142    public String normalizeBaseURL(String baseUrl, String fullUrl) {
143      return fullUrl.substring(baseUrl.length()).replace("/", "");
144    }
145
146    public String getVersionedSystem(String baseSystem, String version) {
147      if (version != null) {
148        return baseSystem + "_" + version;
149      }
150      return baseSystem;
151    }
152  }
153
154  public class CacheToken {
155    private String name;
156    private String key;
157    private String request;
158    public void setName(String n) {
159      String systemName = getSystemNameKeyGenerator().getNameForSystem(n);
160      if (name == null)
161        name = systemName;
162      else if (!systemName.equals(name))
163        name = NAME_FOR_NO_SYSTEM;
164    }
165
166    public String getName() {
167      return name;
168    }
169  }
170
171  private class CacheEntry {
172    private String request;
173    private boolean persistent;
174    private ValidationResult v;
175    private ValueSetExpansionOutcome e;
176  }
177  
178  private class NamedCache {
179    private String name; 
180    private List<CacheEntry> list = new ArrayList<CacheEntry>(); // persistent entries
181    private Map<String, CacheEntry> map = new HashMap<String, CacheEntry>();
182  }
183  
184
185  private Object lock;
186  private String folder;
187  private Map<String, NamedCache> caches = new HashMap<String, NamedCache>();
188  
189  // use lock from the context
190  public TerminologyCache(Object lock, String folder) throws FileNotFoundException, IOException, FHIRException {
191    super();
192    this.lock = lock;
193    this.folder = folder;
194    if (folder != null)
195      load();
196  }
197  
198  public CacheToken generateValidationToken(ValidationOptions options, Coding code, ValueSet vs) {
199    CacheToken ct = new CacheToken();
200    if (code.hasSystem())
201      ct.name = getSystemNameKeyGenerator().getNameForSystem(code.getSystem());
202    else
203      ct.name = NAME_FOR_NO_SYSTEM;
204    JsonParser json = new JsonParser();
205    json.setOutputStyle(OutputStyle.PRETTY);
206    ValueSet vsc = getVSEssense(vs);
207    try {
208      ct.request = "{\"code\" : "+json.composeString(code, "code")+", \"valueSet\" :"+(vsc == null ? "null" : json.composeString(vsc))+(options == null ? "" : ", "+options.toJson())+"}";
209    } catch (IOException e) {
210      throw new Error(e);
211    }
212    ct.key = String.valueOf(hashNWS(ct.request));
213    return ct;
214  }
215
216  public CacheToken generateValidationToken(ValidationOptions options, CodeableConcept code, ValueSet vs) {
217    CacheToken ct = new CacheToken();
218    for (Coding c : code.getCoding()) {
219      if (c.hasSystem())
220        ct.setName(c.getSystem());
221    }
222    JsonParser json = new JsonParser();
223    json.setOutputStyle(OutputStyle.PRETTY);
224    ValueSet vsc = getVSEssense(vs);
225    try {
226      ct.request = "{\"code\" : "+json.composeString(code, "codeableConcept")+", \"valueSet\" :"+json.composeString(vsc)+(options == null ? "" : ", "+options.toJson())+"}";
227    } catch (IOException e) {
228      throw new Error(e);
229    }
230    ct.key = String.valueOf(hashNWS(ct.request));
231    return ct;
232  }
233  
234  public ValueSet getVSEssense(ValueSet vs) {
235    if (vs == null)
236      return null;
237    ValueSet vsc = new ValueSet();
238    vsc.setCompose(vs.getCompose());
239    if (vs.hasExpansion()) {
240      vsc.getExpansion().getParameter().addAll(vs.getExpansion().getParameter());
241      vsc.getExpansion().getContains().addAll(vs.getExpansion().getContains());
242    }
243    return vsc;
244  }
245
246  public CacheToken generateExpandToken(ValueSet vs, boolean heirarchical) {
247    CacheToken ct = new CacheToken();
248    ValueSet vsc = getVSEssense(vs);
249    for (ConceptSetComponent inc : vs.getCompose().getInclude())
250      if (inc.hasSystem())
251        ct.setName(inc.getSystem());
252    for (ConceptSetComponent inc : vs.getCompose().getExclude())
253      if (inc.hasSystem())
254        ct.setName(inc.getSystem());
255    for (ValueSetExpansionContainsComponent inc : vs.getExpansion().getContains())
256      if (inc.hasSystem())
257        ct.setName(inc.getSystem());
258    JsonParser json = new JsonParser();
259    json.setOutputStyle(OutputStyle.PRETTY);
260    try {
261      ct.request = "{\"hierarchical\" : "+(heirarchical ? "true" : "false")+", \"valueSet\" :"+json.composeString(vsc)+"}\r\n";
262    } catch (IOException e) {
263      throw new Error(e);
264    }
265    ct.key = String.valueOf(hashNWS(ct.request));
266    return ct;
267  }
268
269  public NamedCache getNamedCache(CacheToken cacheToken) {
270    NamedCache nc = caches.get(cacheToken.name);
271    if (nc == null) {
272      nc = new NamedCache();
273      nc.name = cacheToken.name;
274      caches.put(nc.name, nc);
275    }
276    return nc;
277  }
278  
279  public ValueSetExpansionOutcome getExpansion(CacheToken cacheToken) {
280    synchronized (lock) {
281      NamedCache nc = getNamedCache(cacheToken);
282      CacheEntry e = nc.map.get(cacheToken.key);
283      if (e == null)
284        return null;
285      else
286        return e.e;
287    }
288  }
289
290  public void cacheExpansion(CacheToken cacheToken, ValueSetExpansionOutcome res, boolean persistent) {
291    synchronized (lock) {      
292      NamedCache nc = getNamedCache(cacheToken);
293      CacheEntry e = new CacheEntry();
294      e.request = cacheToken.request;
295      e.persistent = persistent;
296      e.e = res;
297      store(cacheToken, persistent, nc, e);
298    }    
299  }
300
301  public void store(CacheToken cacheToken, boolean persistent, NamedCache nc, CacheEntry e) {
302    boolean n = nc.map.containsKey(cacheToken.key);
303    nc.map.put(cacheToken.key, e);
304    if (persistent) {
305      if (n) {
306        for (int i = nc.list.size()- 1; i>= 0; i--) {
307          if (nc.list.get(i).request.equals(e.request)) {
308            nc.list.remove(i);
309          }
310        }
311      }
312      nc.list.add(e);
313      save(nc);  
314    }
315  }
316
317  public ValidationResult getValidation(CacheToken cacheToken) {
318    synchronized (lock) {
319      NamedCache nc = getNamedCache(cacheToken);
320      CacheEntry e = nc.map.get(cacheToken.key);
321      if (e == null)
322        return null;
323      else
324        return e.v;
325    }
326  }
327
328  public void cacheValidation(CacheToken cacheToken, ValidationResult res, boolean persistent) {
329    synchronized (lock) {      
330      NamedCache nc = getNamedCache(cacheToken);
331      CacheEntry e = new CacheEntry();
332      e.request = cacheToken.request;
333      e.persistent = persistent;
334      e.v = res;
335      store(cacheToken, persistent, nc, e);
336    }    
337  }
338
339  
340  // persistence
341  
342  public void save() {
343    
344  }
345  
346  private void save(NamedCache nc) {
347    if (folder == null)
348      return;
349    
350    try {
351      OutputStreamWriter sw = new OutputStreamWriter(new FileOutputStream(Utilities.path(folder, nc.name+".cache")), "UTF-8");
352      sw.write(ENTRY_MARKER+"\r\n");
353      JsonParser json = new JsonParser();
354      json.setOutputStyle(OutputStyle.PRETTY);
355      for (CacheEntry ce : nc.list) {
356        sw.write(ce.request.trim());
357        sw.write(BREAK+"\r\n");
358        if (ce.e != null) {
359          sw.write("e: {\r\n");
360          if (ce.e.getValueset() != null)
361            sw.write("  \"valueSet\" : "+json.composeString(ce.e.getValueset()).trim()+",\r\n");
362          sw.write("  \"error\" : \""+Utilities.escapeJson(ce.e.getError()).trim()+"\"\r\n}\r\n");
363        } else {
364          sw.write("v: {\r\n");
365          sw.write("  \"display\" : \""+Utilities.escapeJson(ce.v.getDisplay()).trim()+"\",\r\n");
366          sw.write("  \"severity\" : "+(ce.v.getSeverity() == null ? "null" : "\""+ce.v.getSeverity().toCode().trim()+"\"")+",\r\n");
367          sw.write("  \"error\" : \""+Utilities.escapeJson(ce.v.getMessage()).trim()+"\"\r\n}\r\n");
368        }
369        sw.write(ENTRY_MARKER+"\r\n");
370      }      
371      sw.close();
372    } catch (Exception e) {
373      System.out.println("error saving "+nc.name+": "+e.getMessage());
374    }
375  }
376
377  private void load() throws FHIRException {
378    for (String fn : new File(folder).list()) {
379      if (fn.endsWith(".cache") && !fn.equals("validation.cache")) {
380        try {
381          //  System.out.println("Load "+fn);
382          String title = fn.substring(0, fn.lastIndexOf("."));
383          NamedCache nc = new NamedCache();
384          nc.name = title;
385          caches.put(title, nc);
386          System.out.print(" - load "+title+".cache");
387          String src = TextFile.fileToString(Utilities.path(folder, fn));
388          if (src.startsWith("?"))
389            src = src.substring(1);
390          int i = src.indexOf(ENTRY_MARKER); 
391          while (i > -1) {
392            String s = src.substring(0, i);
393            System.out.print(".");
394            src = src.substring(i+ENTRY_MARKER.length()+1);
395            i = src.indexOf(ENTRY_MARKER);
396            if (!Utilities.noString(s)) {
397              int j = s.indexOf(BREAK);
398              String q = s.substring(0, j);
399              String p = s.substring(j+BREAK.length()+1).trim();
400              CacheEntry ce = new CacheEntry();
401              ce.persistent = true;
402              ce.request = q;
403              boolean e = p.charAt(0) == 'e';
404              p = p.substring(3);
405              JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p);
406              String error = loadJS(o.get("error"));
407              if (e) {
408                if (o.has("valueSet"))
409                  ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")), error, TerminologyServiceErrorClass.UNKNOWN);
410                else
411                  ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN);
412              } else {
413                IssueSeverity severity = o.get("severity") instanceof JsonNull ? null :  IssueSeverity.fromCode(o.get("severity").getAsString());
414                String display = loadJS(o.get("display"));
415                ce.v = new ValidationResult(severity, error, new ConceptDefinitionComponent().setDisplay(display));
416              }
417              nc.map.put(String.valueOf(hashNWS(ce.request)), ce);
418              nc.list.add(ce);
419            }
420          }        
421          System.out.println("done");
422        } catch (Exception e) {
423          throw new FHIRException("Error loading "+fn+": "+e.getMessage(), e);
424        }
425      }
426    }
427  }
428  
429  private String loadJS(JsonElement e) {
430    if (e == null)
431      return null;
432    if (!(e instanceof JsonPrimitive))
433      return null;
434    String s = e.getAsString();
435    if ("".equals(s))
436      return null;
437    return s;
438  }
439
440  private String hashNWS(String s) {
441    s = StringUtils.remove(s, ' ');
442    s = StringUtils.remove(s, '\n');
443    s = StringUtils.remove(s, '\r');
444    return String.valueOf(s.hashCode());
445  }
446
447  // management
448  
449  public TerminologyCache copy() {
450    // TODO Auto-generated method stub
451    return null;
452  }
453  
454  public String summary(ValueSet vs) {
455    if (vs == null)
456      return "null";
457    
458    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
459    for (ConceptSetComponent cc : vs.getCompose().getInclude())
460      b.append("Include "+getIncSummary(cc));
461    for (ConceptSetComponent cc : vs.getCompose().getExclude())
462      b.append("Exclude "+getIncSummary(cc));
463    return b.toString();
464  }
465
466  private String getIncSummary(ConceptSetComponent cc) {
467    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
468    for (UriType vs : cc.getValueSet())
469      b.append(vs.asStringValue());
470    String vsd = b.length() > 0 ? " where the codes are in the value sets ("+b.toString()+")" : "";
471    String system = cc.getSystem();
472    if (cc.hasConcept())
473      return Integer.toString(cc.getConcept().size())+" codes from "+system+vsd;
474    if (cc.hasFilter()) {
475      String s = "";
476      for (ConceptSetFilterComponent f : cc.getFilter()) {
477        if (!Utilities.noString(s))
478          s = s + " & ";
479        s = s + f.getProperty()+" "+f.getOp().toCode()+" "+f.getValue();
480      }
481      return "from "+system+" where "+s+vsd;
482    }
483    return "All codes from "+system+vsd;
484  }
485
486  public String summary(Coding code) {
487    return code.getSystem()+"#"+code.getCode()+": \""+code.getDisplay()+"\"";
488  }
489
490
491  public String summary(CodeableConcept code) {
492    StringBuilder b = new StringBuilder();
493    b.append("{");
494    boolean first = true;
495    for (Coding c : code.getCoding()) {
496      if (first) first = false; else b.append(",");
497      b.append(summary(c));
498    }
499    b.append("}: \"");
500    b.append(code.getText());
501    b.append("\"");
502    return b.toString();
503  }
504  
505}