1# 2# Copyright BitBake Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7import logging 8import os.path 9import errno 10import prserv 11import time 12 13try: 14 import sqlite3 15except ImportError: 16 from pysqlite2 import dbapi2 as sqlite3 17 18logger = logging.getLogger("BitBake.PRserv") 19 20sqlversion = sqlite3.sqlite_version_info 21if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): 22 raise Exception("sqlite3 version 3.3.0 or later is required.") 23 24# 25# "No History" mode - for a given query tuple (version, pkgarch, checksum), 26# the returned value will be the largest among all the values of the same 27# (version, pkgarch). This means the PR value returned can NOT be decremented. 28# 29# "History" mode - Return a new higher value for previously unseen query 30# tuple (version, pkgarch, checksum), otherwise return historical value. 31# Value can decrement if returning to a previous build. 32# 33 34class PRTable(object): 35 def __init__(self, conn, table, nohist, read_only): 36 self.conn = conn 37 self.nohist = nohist 38 self.read_only = read_only 39 self.dirty = False 40 if nohist: 41 self.table = "%s_nohist" % table 42 else: 43 self.table = "%s_hist" % table 44 45 if self.read_only: 46 table_exists = self._execute( 47 "SELECT count(*) FROM sqlite_master \ 48 WHERE type='table' AND name='%s'" % (self.table)) 49 if not table_exists: 50 raise prserv.NotFoundError 51 else: 52 self._execute("CREATE TABLE IF NOT EXISTS %s \ 53 (version TEXT NOT NULL, \ 54 pkgarch TEXT NOT NULL, \ 55 checksum TEXT NOT NULL, \ 56 value INTEGER, \ 57 PRIMARY KEY (version, pkgarch, checksum));" % self.table) 58 59 def _execute(self, *query): 60 """Execute a query, waiting to acquire a lock if necessary""" 61 start = time.time() 62 end = start + 20 63 while True: 64 try: 65 return self.conn.execute(*query) 66 except sqlite3.OperationalError as exc: 67 if 'is locked' in str(exc) and end > time.time(): 68 continue 69 raise exc 70 71 def sync(self): 72 if not self.read_only: 73 self.conn.commit() 74 self._execute("BEGIN EXCLUSIVE TRANSACTION") 75 76 def sync_if_dirty(self): 77 if self.dirty: 78 self.sync() 79 self.dirty = False 80 81 def _getValueHist(self, version, pkgarch, checksum): 82 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 83 (version, pkgarch, checksum)) 84 row=data.fetchone() 85 if row is not None: 86 return row[0] 87 else: 88 #no value found, try to insert 89 if self.read_only: 90 data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table), 91 (version, pkgarch)) 92 row = data.fetchone() 93 if row is not None: 94 return row[0] 95 else: 96 return 0 97 98 try: 99 self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));" 100 % (self.table,self.table), 101 (version,pkgarch, checksum,version, pkgarch)) 102 except sqlite3.IntegrityError as exc: 103 logger.error(str(exc)) 104 105 self.dirty = True 106 107 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 108 (version, pkgarch, checksum)) 109 row=data.fetchone() 110 if row is not None: 111 return row[0] 112 else: 113 raise prserv.NotFoundError 114 115 def _getValueNohist(self, version, pkgarch, checksum): 116 data=self._execute("SELECT value FROM %s \ 117 WHERE version=? AND pkgarch=? AND checksum=? AND \ 118 value >= (select max(value) from %s where version=? AND pkgarch=?);" 119 % (self.table, self.table), 120 (version, pkgarch, checksum, version, pkgarch)) 121 row=data.fetchone() 122 if row is not None: 123 return row[0] 124 else: 125 #no value found, try to insert 126 if self.read_only: 127 data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table), 128 (version, pkgarch)) 129 row = data.fetchone() 130 if row is not None: 131 return row[0] 132 else: 133 return 0 134 135 try: 136 self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));" 137 % (self.table,self.table), 138 (version, pkgarch, checksum, version, pkgarch)) 139 except sqlite3.IntegrityError as exc: 140 logger.error(str(exc)) 141 self.conn.rollback() 142 143 self.dirty = True 144 145 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 146 (version, pkgarch, checksum)) 147 row=data.fetchone() 148 if row is not None: 149 return row[0] 150 else: 151 raise prserv.NotFoundError 152 153 def getValue(self, version, pkgarch, checksum): 154 if self.nohist: 155 return self._getValueNohist(version, pkgarch, checksum) 156 else: 157 return self._getValueHist(version, pkgarch, checksum) 158 159 def _importHist(self, version, pkgarch, checksum, value): 160 if self.read_only: 161 return None 162 163 val = None 164 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 165 (version, pkgarch, checksum)) 166 row = data.fetchone() 167 if row is not None: 168 val=row[0] 169 else: 170 #no value found, try to insert 171 try: 172 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), 173 (version, pkgarch, checksum, value)) 174 except sqlite3.IntegrityError as exc: 175 logger.error(str(exc)) 176 177 self.dirty = True 178 179 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 180 (version, pkgarch, checksum)) 181 row = data.fetchone() 182 if row is not None: 183 val = row[0] 184 return val 185 186 def _importNohist(self, version, pkgarch, checksum, value): 187 if self.read_only: 188 return None 189 190 try: 191 #try to insert 192 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), 193 (version, pkgarch, checksum,value)) 194 except sqlite3.IntegrityError as exc: 195 #already have the record, try to update 196 try: 197 self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?" 198 % (self.table), 199 (value,version,pkgarch,checksum,value)) 200 except sqlite3.IntegrityError as exc: 201 logger.error(str(exc)) 202 203 self.dirty = True 204 205 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table, 206 (version,pkgarch,checksum,value)) 207 row=data.fetchone() 208 if row is not None: 209 return row[0] 210 else: 211 return None 212 213 def importone(self, version, pkgarch, checksum, value): 214 if self.nohist: 215 return self._importNohist(version, pkgarch, checksum, value) 216 else: 217 return self._importHist(version, pkgarch, checksum, value) 218 219 def export(self, version, pkgarch, checksum, colinfo): 220 metainfo = {} 221 #column info 222 if colinfo: 223 metainfo['tbl_name'] = self.table 224 metainfo['core_ver'] = prserv.__version__ 225 metainfo['col_info'] = [] 226 data = self._execute("PRAGMA table_info(%s);" % self.table) 227 for row in data: 228 col = {} 229 col['name'] = row['name'] 230 col['type'] = row['type'] 231 col['notnull'] = row['notnull'] 232 col['dflt_value'] = row['dflt_value'] 233 col['pk'] = row['pk'] 234 metainfo['col_info'].append(col) 235 236 #data info 237 datainfo = [] 238 239 if self.nohist: 240 sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \ 241 (SELECT version,pkgarch,max(value) as maxvalue FROM %s GROUP BY version,pkgarch) as T2 \ 242 WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table) 243 else: 244 sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table 245 sqlarg = [] 246 where = "" 247 if version: 248 where += "AND T1.version=? " 249 sqlarg.append(str(version)) 250 if pkgarch: 251 where += "AND T1.pkgarch=? " 252 sqlarg.append(str(pkgarch)) 253 if checksum: 254 where += "AND T1.checksum=? " 255 sqlarg.append(str(checksum)) 256 257 sqlstmt += where + ";" 258 259 if len(sqlarg): 260 data = self._execute(sqlstmt, tuple(sqlarg)) 261 else: 262 data = self._execute(sqlstmt) 263 for row in data: 264 if row['version']: 265 col = {} 266 col['version'] = row['version'] 267 col['pkgarch'] = row['pkgarch'] 268 col['checksum'] = row['checksum'] 269 col['value'] = row['value'] 270 datainfo.append(col) 271 return (metainfo, datainfo) 272 273 def dump_db(self, fd): 274 writeCount = 0 275 for line in self.conn.iterdump(): 276 writeCount = writeCount + len(line) + 1 277 fd.write(line) 278 fd.write('\n') 279 return writeCount 280 281class PRData(object): 282 """Object representing the PR database""" 283 def __init__(self, filename, nohist=True, read_only=False): 284 self.filename=os.path.abspath(filename) 285 self.nohist=nohist 286 self.read_only = read_only 287 #build directory hierarchy 288 try: 289 os.makedirs(os.path.dirname(self.filename)) 290 except OSError as e: 291 if e.errno != errno.EEXIST: 292 raise e 293 uri = "file:%s%s" % (self.filename, "?mode=ro" if self.read_only else "") 294 logger.debug("Opening PRServ database '%s'" % (uri)) 295 self.connection=sqlite3.connect(uri, uri=True, isolation_level="EXCLUSIVE", check_same_thread = False) 296 self.connection.row_factory=sqlite3.Row 297 if not self.read_only: 298 self.connection.execute("pragma synchronous = off;") 299 self.connection.execute("PRAGMA journal_mode = MEMORY;") 300 self._tables={} 301 302 def disconnect(self): 303 self.connection.close() 304 305 def __getitem__(self,tblname): 306 if not isinstance(tblname, str): 307 raise TypeError("tblname argument must be a string, not '%s'" % 308 type(tblname)) 309 if tblname in self._tables: 310 return self._tables[tblname] 311 else: 312 tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist, self.read_only) 313 return tableobj 314 315 def __delitem__(self, tblname): 316 if tblname in self._tables: 317 del self._tables[tblname] 318 logger.info("drop table %s" % (tblname)) 319 self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) 320