001    /*
002     @license.text@ 
003     */
004    package biz.hammurapi.cache;
005    
006    import java.io.ByteArrayOutputStream;
007    import java.io.File;
008    import java.io.FileInputStream;
009    import java.io.FileOutputStream;
010    import java.io.IOException;
011    import java.io.InputStreamReader;
012    import java.io.ObjectInputStream;
013    import java.io.ObjectOutputStream;
014    import java.io.Serializable;
015    import java.sql.SQLException;
016    import java.util.ArrayList;
017    import java.util.HashSet;
018    import java.util.Iterator;
019    import java.util.List;
020    import java.util.Set;
021    import java.util.Timer;
022    import java.util.TimerTask;
023    
024    import biz.hammurapi.CarryOverException;
025    import biz.hammurapi.RuntimeException;
026    import biz.hammurapi.cache.AbstractProducer;
027    import biz.hammurapi.cache.Cache;
028    import biz.hammurapi.cache.Entry;
029    import biz.hammurapi.cache.Producer;
030    import biz.hammurapi.cache.sql.CacheEntry;
031    import biz.hammurapi.cache.sql.FileCacheEngine;
032    import biz.hammurapi.config.ConfigurationException;
033    import biz.hammurapi.sql.DataIterator;
034    import biz.hammurapi.sql.SQLProcessor;
035    import biz.hammurapi.sql.SQLRuntimeException;
036    import biz.hammurapi.sql.Transaction;
037    import biz.hammurapi.sql.hypersonic.HypersonicStandaloneDataSource;
038    import biz.hammurapi.util.Acceptor;
039    
040    
041    /**
042     * Caches objects in files.
043     * @author Pavel Vlasov
044     * @version $Revision: 1.7 $
045     */
046    public class FileCache extends AbstractProducer implements Cache {
047    
048            private static final int CLEANUP_INTERVAL = 10*60*1000;
049            private static final int MAX_FILES_PER_DIRECTORY = 10000;
050            private static final int ACTIVE_DIRS = 5;
051            private HypersonicStandaloneDataSource ds;
052            private FileCacheEngine engine; 
053            private File dataDir;
054            private long size;
055            private long maxSize;
056            private Producer producer;
057                    
058            private class DirectoryInfo {
059                    File directory;
060                    int fileCount;
061                                    
062                    DirectoryInfo() {
063                            synchronized (FileCache.this) {
064                                    do {
065                                            directory=new File(dataDir, Long.toString(System.currentTimeMillis(), Character.MAX_RADIX));
066                                    } while (directory.exists());
067                                    
068                                    directory.mkdir();
069                                    fileCount=directory.listFiles().length;
070                            }
071                    }
072                    
073                    DirectoryInfo(File dir) {
074                            directory=dir;
075                            fileCount=directory.listFiles().length;
076                    }
077                    
078                    File nextFile() throws IOException {
079                            fileCount++;
080                            return File.createTempFile("cache_", ".tmp", directory);
081                    }
082            }
083            
084            private List directories=new ArrayList();
085            
086            private File nextFile() throws IOException {
087                    Iterator it=directories.iterator();
088                    while (it.hasNext()) {
089                            DirectoryInfo di=(DirectoryInfo) it.next();
090                            if (di.fileCount>MAX_FILES_PER_DIRECTORY) {
091                                    it.remove();
092                            }
093                    }
094                    
095                    while (directories.size()<ACTIVE_DIRS) {
096                            directories.add(new DirectoryInfo());
097                    }
098                    
099                    return ((DirectoryInfo) directories.get(((int) (Math.random() * Integer.MAX_VALUE)) % ACTIVE_DIRS)).nextFile();
100            }
101    
102            /**
103             * 
104             * @param producer
105             * @param dir
106             * @param maxSize - Maximum cache size. Number &lt;=0 means no limit
107             * @throws IOException
108             */
109            public FileCache(Producer producer, File dir, long maxSize) throws IOException {
110                    super();
111                    this.maxSize=maxSize>0 ? maxSize : Long.MAX_VALUE;
112                    this.producer=producer;
113                    if (producer!=null) {
114                            producer.addCache(this);
115                    }
116                    
117                    dataDir=new File(dir, "data");
118                    if (!dataDir.exists()) {
119                            dataDir.mkdir();
120                    }
121                    if (!dataDir.exists()) {
122                            throw new IOException("Cannot create directory "+dataDir.getAbsolutePath());
123                    }
124                    if (!dataDir.isDirectory()) {
125                            throw new IOException("Not a directory: "+dataDir.getAbsolutePath());                   
126                    }
127                    
128                    File[] dirs=dataDir.listFiles();                
129                    for (int i=0; i<dirs.length; i++) {
130                            directories.add(new DirectoryInfo(dirs[i]));
131                            File[] entries=dirs[i].listFiles();             
132                            for (int j=0; j<entries.length; i++) {                               
133                                    size+=entries[i].length();
134                            }
135                    }
136                    
137                    try {
138                            ds=new HypersonicStandaloneDataSource(
139                                            new File(dir, "entries").getAbsolutePath(), 
140                                            new Transaction() {
141    
142                                                    public boolean execute(SQLProcessor processor) throws SQLException {
143                                                            try {
144                                                                    processor.executeScript(new InputStreamReader(getClass().getResourceAsStream("FileCache.sql")));
145                                                            } catch (IOException e) {
146                                                                    throw new CarryOverException(e);
147                                                            }
148                                                            return true;
149                                                    }
150                                            });
151                            
152                            engine = new FileCacheEngine(new SQLProcessor(ds, null));
153                    } catch (ClassNotFoundException e) {
154                            throw new IOException("Caused by "+e);
155                    } catch (SQLException e) {
156                            throw new IOException("Caused by "+e);
157                    } catch (CarryOverException e) {
158                            throw (IOException) e.getCause();
159                    }
160            }
161            
162            private boolean shutDown=false;
163            
164            private void checkShutdown() {
165                    if (shutDown) {
166                            throw new IllegalStateException("Shut down");
167                    }
168            }
169            
170            /**
171             * Shuts down entries database and janitor thread.
172             */
173            public void stop() {
174                    shutDown=true;
175                    ds.shutdown();
176                    janitorTask.cancel();
177                    if (isOwnTimer) {
178                            timer.cancel();
179                    }
180            }
181    
182            synchronized public void put(Object key, Object value, long time, long expirationTime) {
183                    checkShutdown();
184                    if (key instanceof String && value instanceof Serializable) {
185                            try {
186                                    remove(key);
187                                    
188                                    // TODO - modify to avoid running into OS limitations of files per dir.
189                                    File out=nextFile();
190                                    ObjectOutputStream oos=new ObjectOutputStream(new FileOutputStream(out));
191                                    try {
192                                            oos.writeObject(value);
193                                    } finally {
194                                            oos.close();
195                                    }
196                                    
197                                    String fileName = out.getParentFile().getName()+File.separator+out.getName();
198                                    //System.out.println(key + " -> " + fileName);
199                                    engine.insertCacheEntry((String) key, fileName, time, expirationTime, 0);
200                                    size+=out.length();
201                                    
202                                    if (size>maxSize) {
203                                            Iterator it=engine.getCacheEntryLastAccessOrdered().iterator();
204                                            while (size>maxSize && it.hasNext()) {
205                                                    CacheEntry entry=(CacheEntry) it.next();
206                                                    File file = new File(dataDir, entry.getValueFile());
207                                                    long length=file.length();
208                                                    if (file.delete()) {
209                                                            size-=length;
210                                                    }
211                                                    engine.deleteCacheEntry(entry.getEntryKey());                                   
212                                            }
213                                            
214                                            ((DataIterator) it).close();
215                                    }
216                            } catch (IOException e) {
217                                    e.printStackTrace();
218                                    // ignore
219                            } catch (SQLException e) {
220                                    e.printStackTrace();
221                                    // ignore
222                            }
223                    } else {
224                            System.err.println("WARN: Cannot serialize "+value.getClass());
225                    }
226            }
227            
228            private TimerTask janitorTask = new TimerTask() {
229                            
230                            public void run() {
231                                    try {
232                                            long now = System.currentTimeMillis();
233                                            Iterator it=engine.getCacheEntryExpiresLE(now).iterator();
234                                            synchronized (FileCache.this) {
235                                                    while (it.hasNext()) {                          
236                                                            File file = new File(dataDir, ((CacheEntry) it.next()).getValueFile());
237                                                            long length=file.length();
238                                                            if (file.delete()) {
239                                                                    size-=length;
240                                                            }
241                                                    }
242                                            }                                               
243                                            engine.deleteCacheEntryExpiresLE(now);                                          
244                                    } catch (SQLException e) {
245                                            e.printStackTrace();
246                                            // ignore
247                                    } catch (SQLRuntimeException e) {
248                                            e.printStackTrace();
249                                            // ignore
250                                    }
251                            }
252                    };
253    
254            synchronized public void clear() {
255                    checkShutdown();
256                    try {
257                            engine.deleteCacheEntry();
258                    } catch (SQLException e) {
259                             e.printStackTrace();
260                            // ignore
261                    }
262                    File[] entries=dataDir.listFiles();
263                    for (int i=0; i<entries.length; i++) {
264                            long length=entries[i].length();
265                            if (entries[i].delete()) {
266                                    size-=length;
267                            }
268                    }
269            }
270    
271            synchronized public void remove(Object key) {
272                    checkShutdown();
273                    if (key instanceof String) {
274                            try {
275                                    CacheEntry cacheEntry = engine.getCacheEntry((String) key);
276                                    if (cacheEntry!=null) {
277                                            File file = new File(dataDir, cacheEntry.getValueFile());
278                                            long length=file.length();
279                                            if (file.delete()) {
280                                                    size-=length;
281                                            }
282                                            engine.deleteCacheEntry((String) key);
283                                    }
284                            } catch (SQLException e) {
285                                    e.printStackTrace();
286                            }                       
287                    }
288                    onRemove(key);
289            }
290    
291            public Entry get(Object key) {
292                    checkShutdown();
293                    Entry ret=key instanceof String ? _get((String) key) : null;
294                    if (ret==null && producer!=null) {
295                            ret=producer.get(key);
296                            if (ret!=null) {
297                                    put(key, ret.get(), ret.getTime(), ret.getExpirationTime());
298                            }
299                    }
300                    //System.out.println("*** "+key+" Restored ---> "+ret);
301                    return ret;
302            }
303                    
304            synchronized private Entry _get(String key) {
305                    try {                   
306                            CacheEntry entry=engine.getCacheEntry(key);
307                            if (entry==null) {
308                                    return null; 
309                            }
310                            long now=System.currentTimeMillis();
311                            if (entry.getEntryExpires()<=0 || now<=entry.getEntryExpires()) {
312                                    engine.setLastAccess(now, key);
313                                    File valueFile=new File(dataDir, entry.getValueFile());
314                                    //System.out.println("Restoring "+key+" <- "+entry.getValueFile()+" ("+valueFile.length()+" bytes)");
315                                    ObjectInputStream ois=new ObjectInputStream(new FileInputStream(valueFile));
316                                    try {
317                                            final Object value=ois.readObject();
318                                            final long expires=entry.getEntryExpires();
319                                            final long time=entry.getEntryTime();
320                                            return new Entry() {
321    
322                                                    public long getExpirationTime() {
323                                                            return expires;
324                                                    }
325    
326                                                    public long getTime() {
327                                                            return time;
328                                                    }
329                                                    
330                                                    public Object get() {
331                                                            return value;
332                                                    }                               
333                                            };
334                                    } finally {
335                                            ois.close();
336                                    }
337                            }
338                            return null;
339                    } catch (IOException e) {
340                            e.printStackTrace();
341                            remove(key);
342                            return null;
343                    } catch (ClassNotFoundException e) {
344                            e.printStackTrace();
345                            remove(key);
346                            return null;
347                    } catch (SQLException e) {
348                            e.printStackTrace();
349                            return null;                    
350                    }
351            }
352    
353            public static void main(String[] args) throws Exception {
354                    FileCache fileCache=new FileCache(new Producer() {
355    
356                            public Entry get(Object key) {
357                                    try {
358                                            int size=(int) (Math.random()*20000);
359                                            ByteArrayOutputStream bos=new ByteArrayOutputStream();
360                                            ObjectOutputStream oos=new ObjectOutputStream(bos);
361                                            
362                                            while (bos.size()<size) {
363                                                    oos.writeObject(key+"_at_"+bos.size());                                                                         
364                                            }
365                                            oos.close();
366                                            bos.close();
367                                            
368                                            final byte[] bytes = bos.toByteArray();
369                                            return new Entry() {
370            
371                                                    public long getExpirationTime() {
372                                                            return 0;
373                                                    }
374            
375                                                    public long getTime() {
376                                                            return 0;
377                                                    }
378                                                    
379                                                    public Object get() {
380                                                            return bytes;
381                                                    }
382                                                    
383                                            };
384                                    } catch (IOException e) {
385                                            e.printStackTrace();
386                                            return null;
387                                    }
388                            }
389    
390                            public void addCache(Cache cache) {
391                                // No action
392                            }
393    
394                            public Set keySet() {
395                                    return null;
396                            }
397                            
398                    },
399                    new File("C:\\_fileCacheTest"),
400                    100000);
401                    
402                    for (int i=0; i<300; i++) {
403                            fileCache.get("Key_"+i);
404                    }
405                    
406                    fileCache.stop();
407            }
408    
409            public void remove(Acceptor acceptor) {
410                    checkShutdown();
411                    Iterator it=engine.getCacheEntry().iterator();
412                    try {
413                            CacheEntry ce=(CacheEntry) it.next();
414                            if (acceptor.accept(ce.getEntryKey())) {
415                                    try {
416                                            File file = new File(dataDir, engine.getCacheEntry(ce.getEntryKey()).getValueFile());
417                                            long length=file.length();
418                                            if (file.delete()) {
419                                                    size-=length;
420                                            }
421                                            engine.deleteCacheEntry(ce.getEntryKey());
422                                    } catch (SQLException e) {
423                                            e.printStackTrace();
424                                    }                       
425                            }
426                            onRemove(ce.getEntryKey());
427                    } finally {
428                            try {
429                                    ((DataIterator) it).close();
430                            } catch (SQLException e) {
431                                    throw new RuntimeException(e);
432                            }
433                    }
434            }
435            
436            public Set keySet() {
437                    HashSet ret = new HashSet();
438                    Iterator it=engine.getCacheEntry().iterator();
439                    while (it.hasNext()) {
440                            ret.add(((CacheEntry) it.next()).getEntryKey());
441                    }
442                    
443                    if (producer!=null) {
444                            Set pkeys=producer.keySet();
445                            if (pkeys!=null) {
446                                    ret.addAll(pkeys);
447                            }
448                    }
449                    
450                    return ret;
451            }
452    
453            public boolean isActive() {
454                    return !shutDown;
455            }
456            
457            private Timer timer;
458            private boolean isOwnTimer;
459    
460            public void start() throws ConfigurationException {
461                    if (timer==null) {
462                            timer=new Timer();
463                            isOwnTimer=true;
464                    }
465                    
466                    timer.schedule(janitorTask, CLEANUP_INTERVAL, CLEANUP_INTERVAL);
467            }
468    
469            public void setOwner(Object owner) {
470                    // Nothing
471                    
472            }
473            
474    }