2022-03-22 16:34:28 -06:00
import crypto from 'crypto' ;
import dirTree from 'directory-tree' ;
import fs from 'fs' ;
import log4js from 'log4js' ;
2022-03-22 17:29:07 -06:00
import { Op } from 'sequelize' ;
2022-03-22 16:34:28 -06:00
2022-03-22 17:29:07 -06:00
import { Devices , Drives , DriveSegments } from '../models' ;
2022-03-22 16:34:28 -06:00
import { deleteFolderRecursive } from './storage' ;
const logger = log4js . getLogger ( 'cleanup' ) ;
2022-03-22 17:29:07 -06:00
export const affectedDevices = { } ;
2022-03-22 16:34:28 -06:00
async function deleteBootAndCrashLogs ( ) {
2022-03-22 17:29:07 -06:00
const devices = await Devices . findAll ( ) ;
if ( ! devices ) {
logger . warn ( 'deleteBootAndCrashLogs No devices found' ) ;
2022-03-22 16:34:28 -06:00
return ;
}
for ( let t = 0 ; t < devices . length ; t ++ ) {
const device = devices [ t ] ;
const dongleIdHash = crypto . createHmac ( 'sha256' , process . env . APP _SALT )
. update ( device . dongle _id )
. digest ( 'hex' ) ;
2022-03-22 17:29:07 -06:00
const bootlogDirectoryTree = dirTree ( ` ${ process . env . STORAGE _PATH } ${ device . dongle _id } / ${ dongleIdHash } /boot/ ` , { attributes : [ 'size' ] } ) ;
2022-03-22 16:34:28 -06:00
const bootlogFiles = [ ] ;
if ( bootlogDirectoryTree ) {
for ( let i = 0 ; i < bootlogDirectoryTree . children . length ; i ++ ) {
const timeSplit = bootlogDirectoryTree . children [ i ] . name . replace ( 'boot-' , '' )
. replace ( 'crash-' , '' )
. replace ( '.bz2' , '' )
. split ( '--' ) ;
const timeString = ` ${ timeSplit [ 0 ] } ${ timeSplit [ 1 ] . replace ( /-/g , ':' ) } ` ;
bootlogFiles . push ( {
name : bootlogDirectoryTree . children [ i ] . name ,
size : bootlogDirectoryTree . children [ i ] . size ,
date : Date . parse ( timeString ) ,
path : bootlogDirectoryTree . children [ i ] . path ,
} ) ;
}
bootlogFiles . sort ( ( a , b ) => ( ( a . date < b . date ) ? 1 : - 1 ) ) ;
for ( let c = 5 ; c < bootlogFiles . length ; c ++ ) {
logger . info ( ` deleteBootAndCrashLogs deleting boot log ${ bootlogFiles [ c ] . path } ` ) ;
try {
fs . unlinkSync ( bootlogFiles [ c ] . path ) ;
affectedDevices [ device . dongle _id ] = true ;
} catch ( exception ) {
logger . error ( exception ) ;
}
}
}
2022-03-22 17:29:07 -06:00
const crashlogDirectoryTree = dirTree ( ` ${ process . env . STORAGE _PATH } ${ device . dongle _id } / ${ dongleIdHash } /crash/ ` , { attributes : [ 'size' ] } ) ;
2022-03-22 16:34:28 -06:00
const crashlogFiles = [ ] ;
if ( crashlogDirectoryTree ) {
for ( let i = 0 ; i < crashlogDirectoryTree . children . length ; i ++ ) {
const timeSplit = crashlogDirectoryTree . children [ i ] . name . replace ( 'boot-' , '' )
. replace ( 'crash-' , '' )
. replace ( '.bz2' , '' )
. split ( '--' ) ;
const timeString = ` ${ timeSplit [ 0 ] } ${ timeSplit [ 1 ] . replace ( /-/g , ':' ) } ` ;
crashlogFiles . push ( {
name : crashlogDirectoryTree . children [ i ] . name ,
size : crashlogDirectoryTree . children [ i ] . size ,
date : Date . parse ( timeString ) ,
path : crashlogDirectoryTree . children [ i ] . path ,
} ) ;
}
crashlogFiles . sort ( ( a , b ) => ( ( a . date < b . date ) ? 1 : - 1 ) ) ;
for ( let c = 5 ; c < crashlogFiles . length ; c ++ ) {
logger . info ( ` deleteBootAndCrashLogs deleting crash log ${ crashlogFiles [ c ] . path } ` ) ;
try {
fs . unlinkSync ( crashlogFiles [ c ] . path ) ;
affectedDevices [ device . dongle _id ] = true ;
} catch ( exception ) {
logger . error ( exception ) ;
}
}
}
}
}
async function deleteExpiredDrives ( ) {
const expirationTs = Date . now ( ) - process . env . DEVICE _EXPIRATION _DAYS * 24 * 3600 * 1000 ;
2022-03-22 17:29:07 -06:00
const expiredDrives = Drives . findAll ( {
where : {
is _preserved : false ,
is _deleted : false ,
created : { [ Op . lt ] : expirationTs } ,
} ,
} ) ;
2022-03-22 16:34:28 -06:00
if ( ! expiredDrives ) {
2022-03-22 17:29:07 -06:00
logger . info ( 'deleteExpiredDrives No expired drives found' ) ;
2022-03-22 16:34:28 -06:00
return ;
}
for ( let t = 0 ; t < expiredDrives . length ; t ++ ) {
logger . info ( ` deleteExpiredDrives drive ${ expiredDrives [ t ] . dongle _id } ${ expiredDrives [ t ] . identifier } is older than ${ process . env . DEVICE _EXPIRATION _DAYS } days, set is_deleted=true ` ) ;
await Drives . update (
2022-03-22 17:29:07 -06:00
{ is _deleted : true } ,
2022-03-22 16:34:28 -06:00
{ where : { id : expiredDrives [ t ] . id } } ,
) ;
}
}
async function deleteOverQuotaDrives ( ) {
2022-03-22 17:29:07 -06:00
const devices = await Devices . findAll ( {
where : {
storage _used : { [ Op . gt ] : process . env . DEVICE _STORAGE _QUOTA _MB } ,
} ,
} ) ;
if ( devices === null ) {
logger . info ( 'deleteOverQuotaDrives No over quota devices found' ) ;
2022-03-22 16:34:28 -06:00
return ;
}
for ( let t = 0 ; t < devices . length ; t ++ ) {
2022-03-22 17:29:07 -06:00
const { dongle _id : dongleId } = devices [ t ] ;
const drive = await Drives . findOne ( {
where : {
dongle _id : dongleId ,
is _deleted : false ,
is _preserved : false ,
} ,
2022-04-29 17:39:13 -06:00
order : [ [ 'created' , 'ASC' ] ] ,
2022-03-22 17:29:07 -06:00
} ) ;
2022-03-22 16:34:28 -06:00
2022-03-22 17:29:07 -06:00
if ( drive ) {
logger . info ( ` deleteOverQuotaDrives drive ${ drive . dongle _id } ${ drive . identifier } (normal) is deleted for over-quota ` ) ;
await Drives . update (
{ is _deleted : true } ,
{ where : { id : drive . id } } ,
) ;
} else {
const preservedDrive = await Drives . findOne ( {
where : {
dongle _id : dongleId ,
is _preserved : true ,
is _deleted : false ,
} ,
order : [ [ 'created' , 'ASC' ] ] ,
} ) ;
if ( preservedDrive ) {
logger . info ( ` deleteOverQuotaDrives drive ${ preservedDrive . dongle _id } ${ preservedDrive . identifier } (preserved!) is deleted for over-quota ` ) ;
await Drives . update (
{ is _deleted : true } ,
{ where : { id : preservedDrive . id } } ,
) ;
2022-03-22 16:34:28 -06:00
}
}
}
}
async function removeDeletedDrivesPhysically ( ) {
2022-03-22 17:29:07 -06:00
const deletedDrives = await Drives . findAll ( {
where : {
is _deleted : true ,
is _physically _removed : false ,
} ,
} ) ;
2022-03-22 16:34:28 -06:00
if ( ! deletedDrives ) {
2022-03-22 17:29:07 -06:00
logger . info ( 'removeDeletedDrivesPhysically no deleted drives found' ) ;
2022-03-22 16:34:28 -06:00
return ;
}
for ( let t = 0 ; t < deletedDrives . length ; t ++ ) {
2022-03-22 17:29:07 -06:00
const drive = deletedDrives [ t ] ;
const {
id ,
dongle _id : dongleId ,
identifier ,
} = drive ;
logger . info ( ` removeDeletedDrivesPhysically drive ${ dongleId } ${ identifier } is deleted, remove physical files and clean database ` ) ;
2022-03-22 16:34:28 -06:00
const dongleIdHash = crypto . createHmac ( 'sha256' , process . env . APP _SALT )
2022-03-22 17:29:07 -06:00
. update ( dongleId )
2022-03-22 16:34:28 -06:00
. digest ( 'hex' ) ;
const driveIdentifierHash = crypto . createHmac ( 'sha256' , process . env . APP _SALT )
2022-03-22 17:29:07 -06:00
. update ( identifier )
2022-03-22 16:34:28 -06:00
. digest ( 'hex' ) ;
2022-03-22 17:29:07 -06:00
const drivePath = ` ${ process . env . STORAGE _PATH } ${ dongleId } / ${ dongleIdHash } / ${ driveIdentifierHash } ` ;
logger . info ( ` removeDeletedDrivesPhysically drive ${ dongleId } ${ identifier } storage path is ${ drivePath } ` ) ;
2022-03-22 16:34:28 -06:00
try {
2022-03-22 17:29:07 -06:00
const driveResult = await Drives . update ( {
is _physically _removed : true ,
} , {
where : { id } ,
} ) ;
const driveSegmentResult = await DriveSegments . update ( {
is _physically _removed : true ,
} , {
where : { drive _identifier : identifier , dongle _id : dongleId } ,
} ) ;
if ( driveResult && driveSegmentResult ) {
2022-03-22 16:34:28 -06:00
deleteFolderRecursive ( drivePath , { recursive : true } ) ;
}
2022-03-22 17:29:07 -06:00
affectedDevices [ drive . dongle _id ] = true ;
2022-03-22 16:34:28 -06:00
} catch ( exception ) {
logger . error ( exception ) ;
}
}
}
export async function doCleanup ( ) {
await deleteBootAndCrashLogs ( ) ;
await deleteExpiredDrives ( ) ;
await deleteOverQuotaDrives ( ) ;
await removeDeletedDrivesPhysically ( ) ;
}