xref: /OK3568_Linux_fs/yocto/poky/bitbake/lib/bb/persist_data.py (revision 4882a59341e53eb6f0b4789bf948001014eff981) 
1"""BitBake Persistent Data Store
2
3Used to store data in a central location such that other threads/tasks can
4access them at some future date.  Acts as a convenience wrapper around sqlite,
5currently, providing a key/value store accessed by 'domain'.
6"""
7
8# Copyright (C) 2007        Richard Purdie
9# Copyright (C) 2010        Chris Larson <chris_larson@mentor.com>
10#
11# SPDX-License-Identifier: GPL-2.0-only
12#
13
14import collections
15import collections.abc
16import contextlib
17import functools
18import logging
19import os.path
20import sqlite3
21import sys
22from collections.abc import Mapping
23
24sqlversion = sqlite3.sqlite_version_info
25if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
26    raise Exception("sqlite3 version 3.3.0 or later is required.")
27
28
29logger = logging.getLogger("BitBake.PersistData")
30
31@functools.total_ordering
32class SQLTable(collections.abc.MutableMapping):
33    class _Decorators(object):
34        @staticmethod
35        def retry(*, reconnect=True):
36            """
37            Decorator that restarts a function if a database locked sqlite
38            exception occurs. If reconnect is True, the database connection
39            will be closed and reopened each time a failure occurs
40            """
41            def retry_wrapper(f):
42                def wrap_func(self, *args, **kwargs):
43                    # Reconnect if necessary
44                    if self.connection is None and reconnect:
45                        self.reconnect()
46
47                    count = 0
48                    while True:
49                        try:
50                            return f(self, *args, **kwargs)
51                        except sqlite3.OperationalError as exc:
52                            if count < 500 and ('is locked' in str(exc) or 'locking protocol' in str(exc)):
53                                count = count + 1
54                                if reconnect:
55                                    self.reconnect()
56                                continue
57                            raise
58                return wrap_func
59            return retry_wrapper
60
61        @staticmethod
62        def transaction(f):
63            """
64            Decorator that starts a database transaction and creates a database
65            cursor for performing queries. If no exception is thrown, the
66            database results are committed. If an exception occurs, the database
67            is rolled back. In all cases, the cursor is closed after the
68            function ends.
69
70            Note that the cursor is passed as an extra argument to the function
71            after `self` and before any of the normal arguments
72            """
73            def wrap_func(self, *args, **kwargs):
74                # Context manager will COMMIT the database on success,
75                # or ROLLBACK on an exception
76                with self.connection:
77                    # Automatically close the cursor when done
78                    with contextlib.closing(self.connection.cursor()) as cursor:
79                        return f(self, cursor, *args, **kwargs)
80            return wrap_func
81
82    """Object representing a table/domain in the database"""
83    def __init__(self, cachefile, table):
84        self.cachefile = cachefile
85        self.table = table
86
87        self.connection = None
88        self._execute_single("CREATE TABLE IF NOT EXISTS %s(key TEXT PRIMARY KEY NOT NULL, value TEXT);" % table)
89
90    @_Decorators.retry(reconnect=False)
91    @_Decorators.transaction
92    def _setup_database(self, cursor):
93        cursor.execute("pragma synchronous = off;")
94        # Enable WAL and keep the autocheckpoint length small (the default is
95        # usually 1000). Persistent caches are usually read-mostly, so keeping
96        # this short will keep readers running quickly
97        cursor.execute("pragma journal_mode = WAL;")
98        cursor.execute("pragma wal_autocheckpoint = 100;")
99
100    def reconnect(self):
101        if self.connection is not None:
102            self.connection.close()
103        self.connection = sqlite3.connect(self.cachefile, timeout=5)
104        self.connection.text_factory = str
105        self._setup_database()
106
107    @_Decorators.retry()
108    @_Decorators.transaction
109    def _execute_single(self, cursor, *query):
110        """
111        Executes a single query and discards the results. This correctly closes
112        the database cursor when finished
113        """
114        cursor.execute(*query)
115
116    @_Decorators.retry()
117    def _row_iter(self, f, *query):
118        """
119        Helper function that returns a row iterator. Each time __next__ is
120        called on the iterator, the provided function is evaluated to determine
121        the return value
122        """
123        class CursorIter(object):
124            def __init__(self, cursor):
125                self.cursor = cursor
126
127            def __iter__(self):
128                return self
129
130            def __next__(self):
131                row = self.cursor.fetchone()
132                if row is None:
133                    self.cursor.close()
134                    raise StopIteration
135                return f(row)
136
137            def __enter__(self):
138                return self
139
140            def __exit__(self, typ, value, traceback):
141                self.cursor.close()
142                return False
143
144        cursor = self.connection.cursor()
145        try:
146            cursor.execute(*query)
147            return CursorIter(cursor)
148        except:
149            cursor.close()
150
151    def __enter__(self):
152        self.connection.__enter__()
153        return self
154
155    def __exit__(self, *excinfo):
156        self.connection.__exit__(*excinfo)
157
158    @_Decorators.retry()
159    @_Decorators.transaction
160    def __getitem__(self, cursor, key):
161        cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
162        row = cursor.fetchone()
163        if row is not None:
164            return row[1]
165        raise KeyError(key)
166
167    @_Decorators.retry()
168    @_Decorators.transaction
169    def __delitem__(self, cursor, key):
170        if key not in self:
171            raise KeyError(key)
172        cursor.execute("DELETE from %s where key=?;" % self.table, [key])
173
174    @_Decorators.retry()
175    @_Decorators.transaction
176    def __setitem__(self, cursor, key, value):
177        if not isinstance(key, str):
178            raise TypeError('Only string keys are supported')
179        elif not isinstance(value, str):
180            raise TypeError('Only string values are supported')
181
182        # Ensure the entire transaction (including SELECT) executes under write lock
183        cursor.execute("BEGIN EXCLUSIVE")
184
185        cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
186        row = cursor.fetchone()
187        if row is not None:
188            cursor.execute("UPDATE %s SET value=? WHERE key=?;" % self.table, [value, key])
189        else:
190            cursor.execute("INSERT into %s(key, value) values (?, ?);" % self.table, [key, value])
191
192    @_Decorators.retry()
193    @_Decorators.transaction
194    def __contains__(self, cursor, key):
195        cursor.execute('SELECT * from %s where key=?;' % self.table, [key])
196        return cursor.fetchone() is not None
197
198    @_Decorators.retry()
199    @_Decorators.transaction
200    def __len__(self, cursor):
201        cursor.execute("SELECT COUNT(key) FROM %s;" % self.table)
202        row = cursor.fetchone()
203        if row is not None:
204            return row[0]
205
206    def __iter__(self):
207        return self._row_iter(lambda row: row[0], "SELECT key from %s;" % self.table)
208
209    def __lt__(self, other):
210        if not isinstance(other, Mapping):
211            raise NotImplementedError()
212
213        return len(self) < len(other)
214
215    def get_by_pattern(self, pattern):
216        return self._row_iter(lambda row: row[1], "SELECT * FROM %s WHERE key LIKE ?;" %
217                              self.table, [pattern])
218
219    def values(self):
220        return list(self.itervalues())
221
222    def itervalues(self):
223        return self._row_iter(lambda row: row[0], "SELECT value FROM %s;" %
224                              self.table)
225
226    def items(self):
227        return list(self.iteritems())
228
229    def iteritems(self):
230        return self._row_iter(lambda row: (row[0], row[1]), "SELECT * FROM %s;" %
231                              self.table)
232
233    @_Decorators.retry()
234    @_Decorators.transaction
235    def clear(self, cursor):
236        cursor.execute("DELETE FROM %s;" % self.table)
237
238    def has_key(self, key):
239        return key in self
240
241def persist(domain, d):
242    """Convenience factory for SQLTable objects based upon metadata"""
243    import bb.utils
244    cachedir = (d.getVar("PERSISTENT_DIR") or
245                d.getVar("CACHE"))
246    if not cachedir:
247        logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable")
248        sys.exit(1)
249
250    bb.utils.mkdirhier(cachedir)
251    cachefile = os.path.join(cachedir, "bb_persist_data.sqlite3")
252    return SQLTable(cachefile, domain)
253