001 /* 002 @license.text@ 003 */ 004 package biz.hammurapi.cache; 005 006 import java.io.ByteArrayOutputStream; 007 import java.io.File; 008 import java.io.FileInputStream; 009 import java.io.FileOutputStream; 010 import java.io.IOException; 011 import java.io.InputStreamReader; 012 import java.io.ObjectInputStream; 013 import java.io.ObjectOutputStream; 014 import java.io.Serializable; 015 import java.sql.SQLException; 016 import java.util.ArrayList; 017 import java.util.HashSet; 018 import java.util.Iterator; 019 import java.util.List; 020 import java.util.Set; 021 import java.util.Timer; 022 import java.util.TimerTask; 023 024 import biz.hammurapi.CarryOverException; 025 import biz.hammurapi.RuntimeException; 026 import biz.hammurapi.cache.AbstractProducer; 027 import biz.hammurapi.cache.Cache; 028 import biz.hammurapi.cache.Entry; 029 import biz.hammurapi.cache.Producer; 030 import biz.hammurapi.cache.sql.CacheEntry; 031 import biz.hammurapi.cache.sql.FileCacheEngine; 032 import biz.hammurapi.config.ConfigurationException; 033 import biz.hammurapi.sql.DataIterator; 034 import biz.hammurapi.sql.SQLProcessor; 035 import biz.hammurapi.sql.SQLRuntimeException; 036 import biz.hammurapi.sql.Transaction; 037 import biz.hammurapi.sql.hypersonic.HypersonicStandaloneDataSource; 038 import biz.hammurapi.util.Acceptor; 039 040 041 /** 042 * Caches objects in files. 043 * @author Pavel Vlasov 044 * @version $Revision: 1.7 $ 045 */ 046 public class FileCache extends AbstractProducer implements Cache { 047 048 private static final int CLEANUP_INTERVAL = 10*60*1000; 049 private static final int MAX_FILES_PER_DIRECTORY = 10000; 050 private static final int ACTIVE_DIRS = 5; 051 private HypersonicStandaloneDataSource ds; 052 private FileCacheEngine engine; 053 private File dataDir; 054 private long size; 055 private long maxSize; 056 private Producer producer; 057 058 private class DirectoryInfo { 059 File directory; 060 int fileCount; 061 062 DirectoryInfo() { 063 synchronized (FileCache.this) { 064 do { 065 directory=new File(dataDir, Long.toString(System.currentTimeMillis(), Character.MAX_RADIX)); 066 } while (directory.exists()); 067 068 directory.mkdir(); 069 fileCount=directory.listFiles().length; 070 } 071 } 072 073 DirectoryInfo(File dir) { 074 directory=dir; 075 fileCount=directory.listFiles().length; 076 } 077 078 File nextFile() throws IOException { 079 fileCount++; 080 return File.createTempFile("cache_", ".tmp", directory); 081 } 082 } 083 084 private List directories=new ArrayList(); 085 086 private File nextFile() throws IOException { 087 Iterator it=directories.iterator(); 088 while (it.hasNext()) { 089 DirectoryInfo di=(DirectoryInfo) it.next(); 090 if (di.fileCount>MAX_FILES_PER_DIRECTORY) { 091 it.remove(); 092 } 093 } 094 095 while (directories.size()<ACTIVE_DIRS) { 096 directories.add(new DirectoryInfo()); 097 } 098 099 return ((DirectoryInfo) directories.get(((int) (Math.random() * Integer.MAX_VALUE)) % ACTIVE_DIRS)).nextFile(); 100 } 101 102 /** 103 * 104 * @param producer 105 * @param dir 106 * @param maxSize - Maximum cache size. Number <=0 means no limit 107 * @throws IOException 108 */ 109 public FileCache(Producer producer, File dir, long maxSize) throws IOException { 110 super(); 111 this.maxSize=maxSize>0 ? maxSize : Long.MAX_VALUE; 112 this.producer=producer; 113 if (producer!=null) { 114 producer.addCache(this); 115 } 116 117 dataDir=new File(dir, "data"); 118 if (!dataDir.exists()) { 119 dataDir.mkdir(); 120 } 121 if (!dataDir.exists()) { 122 throw new IOException("Cannot create directory "+dataDir.getAbsolutePath()); 123 } 124 if (!dataDir.isDirectory()) { 125 throw new IOException("Not a directory: "+dataDir.getAbsolutePath()); 126 } 127 128 File[] dirs=dataDir.listFiles(); 129 for (int i=0; i<dirs.length; i++) { 130 directories.add(new DirectoryInfo(dirs[i])); 131 File[] entries=dirs[i].listFiles(); 132 for (int j=0; j<entries.length; i++) { 133 size+=entries[i].length(); 134 } 135 } 136 137 try { 138 ds=new HypersonicStandaloneDataSource( 139 new File(dir, "entries").getAbsolutePath(), 140 new Transaction() { 141 142 public boolean execute(SQLProcessor processor) throws SQLException { 143 try { 144 processor.executeScript(new InputStreamReader(getClass().getResourceAsStream("FileCache.sql"))); 145 } catch (IOException e) { 146 throw new CarryOverException(e); 147 } 148 return true; 149 } 150 }); 151 152 engine = new FileCacheEngine(new SQLProcessor(ds, null)); 153 } catch (ClassNotFoundException e) { 154 throw new IOException("Caused by "+e); 155 } catch (SQLException e) { 156 throw new IOException("Caused by "+e); 157 } catch (CarryOverException e) { 158 throw (IOException) e.getCause(); 159 } 160 } 161 162 private boolean shutDown=false; 163 164 private void checkShutdown() { 165 if (shutDown) { 166 throw new IllegalStateException("Shut down"); 167 } 168 } 169 170 /** 171 * Shuts down entries database and janitor thread. 172 */ 173 public void stop() { 174 shutDown=true; 175 ds.shutdown(); 176 janitorTask.cancel(); 177 if (isOwnTimer) { 178 timer.cancel(); 179 } 180 } 181 182 synchronized public void put(Object key, Object value, long time, long expirationTime) { 183 checkShutdown(); 184 if (key instanceof String && value instanceof Serializable) { 185 try { 186 remove(key); 187 188 // TODO - modify to avoid running into OS limitations of files per dir. 189 File out=nextFile(); 190 ObjectOutputStream oos=new ObjectOutputStream(new FileOutputStream(out)); 191 try { 192 oos.writeObject(value); 193 } finally { 194 oos.close(); 195 } 196 197 String fileName = out.getParentFile().getName()+File.separator+out.getName(); 198 //System.out.println(key + " -> " + fileName); 199 engine.insertCacheEntry((String) key, fileName, time, expirationTime, 0); 200 size+=out.length(); 201 202 if (size>maxSize) { 203 Iterator it=engine.getCacheEntryLastAccessOrdered().iterator(); 204 while (size>maxSize && it.hasNext()) { 205 CacheEntry entry=(CacheEntry) it.next(); 206 File file = new File(dataDir, entry.getValueFile()); 207 long length=file.length(); 208 if (file.delete()) { 209 size-=length; 210 } 211 engine.deleteCacheEntry(entry.getEntryKey()); 212 } 213 214 ((DataIterator) it).close(); 215 } 216 } catch (IOException e) { 217 e.printStackTrace(); 218 // ignore 219 } catch (SQLException e) { 220 e.printStackTrace(); 221 // ignore 222 } 223 } else { 224 System.err.println("WARN: Cannot serialize "+value.getClass()); 225 } 226 } 227 228 private TimerTask janitorTask = new TimerTask() { 229 230 public void run() { 231 try { 232 long now = System.currentTimeMillis(); 233 Iterator it=engine.getCacheEntryExpiresLE(now).iterator(); 234 synchronized (FileCache.this) { 235 while (it.hasNext()) { 236 File file = new File(dataDir, ((CacheEntry) it.next()).getValueFile()); 237 long length=file.length(); 238 if (file.delete()) { 239 size-=length; 240 } 241 } 242 } 243 engine.deleteCacheEntryExpiresLE(now); 244 } catch (SQLException e) { 245 e.printStackTrace(); 246 // ignore 247 } catch (SQLRuntimeException e) { 248 e.printStackTrace(); 249 // ignore 250 } 251 } 252 }; 253 254 synchronized public void clear() { 255 checkShutdown(); 256 try { 257 engine.deleteCacheEntry(); 258 } catch (SQLException e) { 259 e.printStackTrace(); 260 // ignore 261 } 262 File[] entries=dataDir.listFiles(); 263 for (int i=0; i<entries.length; i++) { 264 long length=entries[i].length(); 265 if (entries[i].delete()) { 266 size-=length; 267 } 268 } 269 } 270 271 synchronized public void remove(Object key) { 272 checkShutdown(); 273 if (key instanceof String) { 274 try { 275 CacheEntry cacheEntry = engine.getCacheEntry((String) key); 276 if (cacheEntry!=null) { 277 File file = new File(dataDir, cacheEntry.getValueFile()); 278 long length=file.length(); 279 if (file.delete()) { 280 size-=length; 281 } 282 engine.deleteCacheEntry((String) key); 283 } 284 } catch (SQLException e) { 285 e.printStackTrace(); 286 } 287 } 288 onRemove(key); 289 } 290 291 public Entry get(Object key) { 292 checkShutdown(); 293 Entry ret=key instanceof String ? _get((String) key) : null; 294 if (ret==null && producer!=null) { 295 ret=producer.get(key); 296 if (ret!=null) { 297 put(key, ret.get(), ret.getTime(), ret.getExpirationTime()); 298 } 299 } 300 //System.out.println("*** "+key+" Restored ---> "+ret); 301 return ret; 302 } 303 304 synchronized private Entry _get(String key) { 305 try { 306 CacheEntry entry=engine.getCacheEntry(key); 307 if (entry==null) { 308 return null; 309 } 310 long now=System.currentTimeMillis(); 311 if (entry.getEntryExpires()<=0 || now<=entry.getEntryExpires()) { 312 engine.setLastAccess(now, key); 313 File valueFile=new File(dataDir, entry.getValueFile()); 314 //System.out.println("Restoring "+key+" <- "+entry.getValueFile()+" ("+valueFile.length()+" bytes)"); 315 ObjectInputStream ois=new ObjectInputStream(new FileInputStream(valueFile)); 316 try { 317 final Object value=ois.readObject(); 318 final long expires=entry.getEntryExpires(); 319 final long time=entry.getEntryTime(); 320 return new Entry() { 321 322 public long getExpirationTime() { 323 return expires; 324 } 325 326 public long getTime() { 327 return time; 328 } 329 330 public Object get() { 331 return value; 332 } 333 }; 334 } finally { 335 ois.close(); 336 } 337 } 338 return null; 339 } catch (IOException e) { 340 e.printStackTrace(); 341 remove(key); 342 return null; 343 } catch (ClassNotFoundException e) { 344 e.printStackTrace(); 345 remove(key); 346 return null; 347 } catch (SQLException e) { 348 e.printStackTrace(); 349 return null; 350 } 351 } 352 353 public static void main(String[] args) throws Exception { 354 FileCache fileCache=new FileCache(new Producer() { 355 356 public Entry get(Object key) { 357 try { 358 int size=(int) (Math.random()*20000); 359 ByteArrayOutputStream bos=new ByteArrayOutputStream(); 360 ObjectOutputStream oos=new ObjectOutputStream(bos); 361 362 while (bos.size()<size) { 363 oos.writeObject(key+"_at_"+bos.size()); 364 } 365 oos.close(); 366 bos.close(); 367 368 final byte[] bytes = bos.toByteArray(); 369 return new Entry() { 370 371 public long getExpirationTime() { 372 return 0; 373 } 374 375 public long getTime() { 376 return 0; 377 } 378 379 public Object get() { 380 return bytes; 381 } 382 383 }; 384 } catch (IOException e) { 385 e.printStackTrace(); 386 return null; 387 } 388 } 389 390 public void addCache(Cache cache) { 391 // No action 392 } 393 394 public Set keySet() { 395 return null; 396 } 397 398 }, 399 new File("C:\\_fileCacheTest"), 400 100000); 401 402 for (int i=0; i<300; i++) { 403 fileCache.get("Key_"+i); 404 } 405 406 fileCache.stop(); 407 } 408 409 public void remove(Acceptor acceptor) { 410 checkShutdown(); 411 Iterator it=engine.getCacheEntry().iterator(); 412 try { 413 CacheEntry ce=(CacheEntry) it.next(); 414 if (acceptor.accept(ce.getEntryKey())) { 415 try { 416 File file = new File(dataDir, engine.getCacheEntry(ce.getEntryKey()).getValueFile()); 417 long length=file.length(); 418 if (file.delete()) { 419 size-=length; 420 } 421 engine.deleteCacheEntry(ce.getEntryKey()); 422 } catch (SQLException e) { 423 e.printStackTrace(); 424 } 425 } 426 onRemove(ce.getEntryKey()); 427 } finally { 428 try { 429 ((DataIterator) it).close(); 430 } catch (SQLException e) { 431 throw new RuntimeException(e); 432 } 433 } 434 } 435 436 public Set keySet() { 437 HashSet ret = new HashSet(); 438 Iterator it=engine.getCacheEntry().iterator(); 439 while (it.hasNext()) { 440 ret.add(((CacheEntry) it.next()).getEntryKey()); 441 } 442 443 if (producer!=null) { 444 Set pkeys=producer.keySet(); 445 if (pkeys!=null) { 446 ret.addAll(pkeys); 447 } 448 } 449 450 return ret; 451 } 452 453 public boolean isActive() { 454 return !shutDown; 455 } 456 457 private Timer timer; 458 private boolean isOwnTimer; 459 460 public void start() throws ConfigurationException { 461 if (timer==null) { 462 timer=new Timer(); 463 isOwnTimer=true; 464 } 465 466 timer.schedule(janitorTask, CLEANUP_INTERVAL, CLEANUP_INTERVAL); 467 } 468 469 public void setOwner(Object owner) { 470 // Nothing 471 472 } 473 474 }