1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041 |
- #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
- #include <linux/init.h>
- #include <linux/kernel.h>
- #include <linux/list.h>
- #include <linux/sched.h>
- #include <linux/jiffies.h>
- #include <linux/delay.h>
- #include <linux/export.h>
- #include <linux/kthread.h>
- #include <linux/rbtree.h>
- #include <linux/fs.h>
- #include <linux/debugfs.h>
- #include <linux/seq_file.h>
- #include <linux/cpumask.h>
- #include <linux/spinlock.h>
- #include <linux/mutex.h>
- #include <linux/rcupdate.h>
- #include <linux/stacktrace.h>
- #include <linux/cache.h>
- #include <linux/percpu.h>
- #include <linux/hardirq.h>
- #include <linux/bootmem.h>
- #include <linux/pfn.h>
- #include <linux/mmzone.h>
- #include <linux/slab.h>
- #include <linux/thread_info.h>
- #include <linux/err.h>
- #include <linux/uaccess.h>
- #include <linux/string.h>
- #include <linux/nodemask.h>
- #include <linux/mm.h>
- #include <linux/workqueue.h>
- #include <linux/crc32.h>
- #include <asm/sections.h>
- #include <asm/processor.h>
- #include <linux/atomic.h>
- #include <linux/kasan.h>
- #include <linux/kmemcheck.h>
- #include <linux/kmemleak.h>
- #include <linux/memory_hotplug.h>
- #define MAX_TRACE 16
- #define MSECS_MIN_AGE 5000
- #define SECS_FIRST_SCAN 60
- #define SECS_SCAN_WAIT 600
- #define MAX_SCAN_SIZE 4096
- #define BYTES_PER_POINTER sizeof(void *)
- #define gfp_kmemleak_mask(gfp) (((gfp) & (GFP_KERNEL | GFP_ATOMIC)) | \
- __GFP_NORETRY | __GFP_NOMEMALLOC | \
- __GFP_NOWARN)
- struct kmemleak_scan_area {
- struct hlist_node node;
- unsigned long start;
- size_t size;
- };
- #define KMEMLEAK_GREY 0
- #define KMEMLEAK_BLACK -1
- struct kmemleak_object {
- spinlock_t lock;
- unsigned long flags;
- struct list_head object_list;
- struct list_head gray_list;
- struct rb_node rb_node;
- struct rcu_head rcu;
-
- atomic_t use_count;
- unsigned long pointer;
- size_t size;
-
- int min_count;
-
- int count;
-
- u32 checksum;
-
- struct hlist_head area_list;
- unsigned long trace[MAX_TRACE];
- unsigned int trace_len;
- unsigned long jiffies;
- pid_t pid;
- char comm[TASK_COMM_LEN];
- };
- #define OBJECT_ALLOCATED (1 << 0)
- #define OBJECT_REPORTED (1 << 1)
- #define OBJECT_NO_SCAN (1 << 2)
- #define HEX_ROW_SIZE 16
- #define HEX_GROUP_SIZE 1
- #define HEX_ASCII 1
- #define HEX_MAX_LINES 2
- static LIST_HEAD(object_list);
- static LIST_HEAD(gray_list);
- static struct rb_root object_tree_root = RB_ROOT;
- static DEFINE_RWLOCK(kmemleak_lock);
- static struct kmem_cache *object_cache;
- static struct kmem_cache *scan_area_cache;
- static int kmemleak_enabled;
- static int kmemleak_free_enabled;
- static int kmemleak_initialized;
- static int kmemleak_early_log = 1;
- static int kmemleak_warning;
- static int kmemleak_error;
- static unsigned long min_addr = ULONG_MAX;
- static unsigned long max_addr;
- static struct task_struct *scan_thread;
- static unsigned long jiffies_min_age;
- static unsigned long jiffies_last_scan;
- static signed long jiffies_scan_wait;
- #ifdef CONFIG_DEBUG_TASK_STACK_SCAN_OFF
- static int kmemleak_stack_scan;
- #else
- static int kmemleak_stack_scan = 1;
- #endif
- static DEFINE_MUTEX(scan_mutex);
- static int kmemleak_skip_disable;
- static bool kmemleak_found_leaks;
- enum {
- KMEMLEAK_ALLOC,
- KMEMLEAK_ALLOC_PERCPU,
- KMEMLEAK_FREE,
- KMEMLEAK_FREE_PART,
- KMEMLEAK_FREE_PERCPU,
- KMEMLEAK_NOT_LEAK,
- KMEMLEAK_IGNORE,
- KMEMLEAK_SCAN_AREA,
- KMEMLEAK_NO_SCAN
- };
- struct early_log {
- int op_type;
- const void *ptr;
- size_t size;
- int min_count;
- unsigned long trace[MAX_TRACE];
- unsigned int trace_len;
- };
- static struct early_log
- early_log[CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE] __initdata;
- static int crt_early_log __initdata;
- static void kmemleak_disable(void);
- #define kmemleak_warn(x...) do { \
- pr_warn(x); \
- dump_stack(); \
- kmemleak_warning = 1; \
- } while (0)
- #define kmemleak_stop(x...) do { \
- kmemleak_warn(x); \
- kmemleak_disable(); \
- } while (0)
- static void hex_dump_object(struct seq_file *seq,
- struct kmemleak_object *object)
- {
- const u8 *ptr = (const u8 *)object->pointer;
- size_t len;
-
- len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);
- seq_printf(seq, " hex dump (first %zu bytes):\n", len);
- kasan_disable_current();
- seq_hex_dump(seq, " ", DUMP_PREFIX_NONE, HEX_ROW_SIZE,
- HEX_GROUP_SIZE, ptr, len, HEX_ASCII);
- kasan_enable_current();
- }
- static bool color_white(const struct kmemleak_object *object)
- {
- return object->count != KMEMLEAK_BLACK &&
- object->count < object->min_count;
- }
- static bool color_gray(const struct kmemleak_object *object)
- {
- return object->min_count != KMEMLEAK_BLACK &&
- object->count >= object->min_count;
- }
- static bool unreferenced_object(struct kmemleak_object *object)
- {
- return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&
- time_before_eq(object->jiffies + jiffies_min_age,
- jiffies_last_scan);
- }
- static void print_unreferenced(struct seq_file *seq,
- struct kmemleak_object *object)
- {
- int i;
- unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);
- seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
- object->pointer, object->size);
- seq_printf(seq, " comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",
- object->comm, object->pid, object->jiffies,
- msecs_age / 1000, msecs_age % 1000);
- hex_dump_object(seq, object);
- seq_printf(seq, " backtrace:\n");
- for (i = 0; i < object->trace_len; i++) {
- void *ptr = (void *)object->trace[i];
- seq_printf(seq, " [<%p>] %pS\n", ptr, ptr);
- }
- }
- static void dump_object_info(struct kmemleak_object *object)
- {
- struct stack_trace trace;
- trace.nr_entries = object->trace_len;
- trace.entries = object->trace;
- pr_notice("Object 0x%08lx (size %zu):\n",
- object->pointer, object->size);
- pr_notice(" comm \"%s\", pid %d, jiffies %lu\n",
- object->comm, object->pid, object->jiffies);
- pr_notice(" min_count = %d\n", object->min_count);
- pr_notice(" count = %d\n", object->count);
- pr_notice(" flags = 0x%lx\n", object->flags);
- pr_notice(" checksum = %u\n", object->checksum);
- pr_notice(" backtrace:\n");
- print_stack_trace(&trace, 4);
- }
- static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
- {
- struct rb_node *rb = object_tree_root.rb_node;
- while (rb) {
- struct kmemleak_object *object =
- rb_entry(rb, struct kmemleak_object, rb_node);
- if (ptr < object->pointer)
- rb = object->rb_node.rb_left;
- else if (object->pointer + object->size <= ptr)
- rb = object->rb_node.rb_right;
- else if (object->pointer == ptr || alias)
- return object;
- else {
- kmemleak_warn("Found object by alias at 0x%08lx\n",
- ptr);
- dump_object_info(object);
- break;
- }
- }
- return NULL;
- }
- static int get_object(struct kmemleak_object *object)
- {
- return atomic_inc_not_zero(&object->use_count);
- }
- static void free_object_rcu(struct rcu_head *rcu)
- {
- struct hlist_node *tmp;
- struct kmemleak_scan_area *area;
- struct kmemleak_object *object =
- container_of(rcu, struct kmemleak_object, rcu);
-
- hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {
- hlist_del(&area->node);
- kmem_cache_free(scan_area_cache, area);
- }
- kmem_cache_free(object_cache, object);
- }
- static void put_object(struct kmemleak_object *object)
- {
- if (!atomic_dec_and_test(&object->use_count))
- return;
-
- WARN_ON(object->flags & OBJECT_ALLOCATED);
- call_rcu(&object->rcu, free_object_rcu);
- }
- static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
- {
- unsigned long flags;
- struct kmemleak_object *object;
- rcu_read_lock();
- read_lock_irqsave(&kmemleak_lock, flags);
- object = lookup_object(ptr, alias);
- read_unlock_irqrestore(&kmemleak_lock, flags);
-
- if (object && !get_object(object))
- object = NULL;
- rcu_read_unlock();
- return object;
- }
- static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int alias)
- {
- unsigned long flags;
- struct kmemleak_object *object;
- write_lock_irqsave(&kmemleak_lock, flags);
- object = lookup_object(ptr, alias);
- if (object) {
- rb_erase(&object->rb_node, &object_tree_root);
- list_del_rcu(&object->object_list);
- }
- write_unlock_irqrestore(&kmemleak_lock, flags);
- return object;
- }
- static int __save_stack_trace(unsigned long *trace)
- {
- struct stack_trace stack_trace;
- stack_trace.max_entries = MAX_TRACE;
- stack_trace.nr_entries = 0;
- stack_trace.entries = trace;
- stack_trace.skip = 2;
- save_stack_trace(&stack_trace);
- return stack_trace.nr_entries;
- }
- static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
- int min_count, gfp_t gfp)
- {
- unsigned long flags;
- struct kmemleak_object *object, *parent;
- struct rb_node **link, *rb_parent;
- object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
- if (!object) {
- pr_warn("Cannot allocate a kmemleak_object structure\n");
- kmemleak_disable();
- return NULL;
- }
- INIT_LIST_HEAD(&object->object_list);
- INIT_LIST_HEAD(&object->gray_list);
- INIT_HLIST_HEAD(&object->area_list);
- spin_lock_init(&object->lock);
- atomic_set(&object->use_count, 1);
- object->flags = OBJECT_ALLOCATED;
- object->pointer = ptr;
- object->size = size;
- object->min_count = min_count;
- object->count = 0;
- object->jiffies = jiffies;
- object->checksum = 0;
-
- if (in_irq()) {
- object->pid = 0;
- strncpy(object->comm, "hardirq", sizeof(object->comm));
- } else if (in_serving_softirq()) {
- object->pid = 0;
- strncpy(object->comm, "softirq", sizeof(object->comm));
- } else {
- object->pid = current->pid;
-
- strncpy(object->comm, current->comm, sizeof(object->comm));
- }
-
- object->trace_len = __save_stack_trace(object->trace);
- write_lock_irqsave(&kmemleak_lock, flags);
- min_addr = min(min_addr, ptr);
- max_addr = max(max_addr, ptr + size);
- link = &object_tree_root.rb_node;
- rb_parent = NULL;
- while (*link) {
- rb_parent = *link;
- parent = rb_entry(rb_parent, struct kmemleak_object, rb_node);
- if (ptr + size <= parent->pointer)
- link = &parent->rb_node.rb_left;
- else if (parent->pointer + parent->size <= ptr)
- link = &parent->rb_node.rb_right;
- else {
- kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",
- ptr);
-
- dump_object_info(parent);
- kmem_cache_free(object_cache, object);
- object = NULL;
- goto out;
- }
- }
- rb_link_node(&object->rb_node, rb_parent, link);
- rb_insert_color(&object->rb_node, &object_tree_root);
- list_add_tail_rcu(&object->object_list, &object_list);
- out:
- write_unlock_irqrestore(&kmemleak_lock, flags);
- return object;
- }
- static void __delete_object(struct kmemleak_object *object)
- {
- unsigned long flags;
- WARN_ON(!(object->flags & OBJECT_ALLOCATED));
- WARN_ON(atomic_read(&object->use_count) < 1);
-
- spin_lock_irqsave(&object->lock, flags);
- object->flags &= ~OBJECT_ALLOCATED;
- spin_unlock_irqrestore(&object->lock, flags);
- put_object(object);
- }
- static void delete_object_full(unsigned long ptr)
- {
- struct kmemleak_object *object;
- object = find_and_remove_object(ptr, 0);
- if (!object) {
- #ifdef DEBUG
- kmemleak_warn("Freeing unknown object at 0x%08lx\n",
- ptr);
- #endif
- return;
- }
- __delete_object(object);
- }
- static void delete_object_part(unsigned long ptr, size_t size)
- {
- struct kmemleak_object *object;
- unsigned long start, end;
- object = find_and_remove_object(ptr, 1);
- if (!object) {
- #ifdef DEBUG
- kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
- ptr, size);
- #endif
- return;
- }
-
- start = object->pointer;
- end = object->pointer + object->size;
- if (ptr > start)
- create_object(start, ptr - start, object->min_count,
- GFP_KERNEL);
- if (ptr + size < end)
- create_object(ptr + size, end - ptr - size, object->min_count,
- GFP_KERNEL);
- __delete_object(object);
- }
- static void __paint_it(struct kmemleak_object *object, int color)
- {
- object->min_count = color;
- if (color == KMEMLEAK_BLACK)
- object->flags |= OBJECT_NO_SCAN;
- }
- static void paint_it(struct kmemleak_object *object, int color)
- {
- unsigned long flags;
- spin_lock_irqsave(&object->lock, flags);
- __paint_it(object, color);
- spin_unlock_irqrestore(&object->lock, flags);
- }
- static void paint_ptr(unsigned long ptr, int color)
- {
- struct kmemleak_object *object;
- object = find_and_get_object(ptr, 0);
- if (!object) {
- kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
- ptr,
- (color == KMEMLEAK_GREY) ? "Grey" :
- (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
- return;
- }
- paint_it(object, color);
- put_object(object);
- }
- static void make_gray_object(unsigned long ptr)
- {
- paint_ptr(ptr, KMEMLEAK_GREY);
- }
- static void make_black_object(unsigned long ptr)
- {
- paint_ptr(ptr, KMEMLEAK_BLACK);
- }
- static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
- {
- unsigned long flags;
- struct kmemleak_object *object;
- struct kmemleak_scan_area *area;
- object = find_and_get_object(ptr, 1);
- if (!object) {
- kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
- ptr);
- return;
- }
- area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
- if (!area) {
- pr_warn("Cannot allocate a scan area\n");
- goto out;
- }
- spin_lock_irqsave(&object->lock, flags);
- if (size == SIZE_MAX) {
- size = object->pointer + object->size - ptr;
- } else if (ptr + size > object->pointer + object->size) {
- kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
- dump_object_info(object);
- kmem_cache_free(scan_area_cache, area);
- goto out_unlock;
- }
- INIT_HLIST_NODE(&area->node);
- area->start = ptr;
- area->size = size;
- hlist_add_head(&area->node, &object->area_list);
- out_unlock:
- spin_unlock_irqrestore(&object->lock, flags);
- out:
- put_object(object);
- }
- static void object_no_scan(unsigned long ptr)
- {
- unsigned long flags;
- struct kmemleak_object *object;
- object = find_and_get_object(ptr, 0);
- if (!object) {
- kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
- return;
- }
- spin_lock_irqsave(&object->lock, flags);
- object->flags |= OBJECT_NO_SCAN;
- spin_unlock_irqrestore(&object->lock, flags);
- put_object(object);
- }
- static void __init log_early(int op_type, const void *ptr, size_t size,
- int min_count)
- {
- unsigned long flags;
- struct early_log *log;
- if (kmemleak_error) {
-
- crt_early_log++;
- return;
- }
- if (crt_early_log >= ARRAY_SIZE(early_log)) {
- crt_early_log++;
- kmemleak_disable();
- return;
- }
-
- local_irq_save(flags);
- log = &early_log[crt_early_log];
- log->op_type = op_type;
- log->ptr = ptr;
- log->size = size;
- log->min_count = min_count;
- log->trace_len = __save_stack_trace(log->trace);
- crt_early_log++;
- local_irq_restore(flags);
- }
- static void early_alloc(struct early_log *log)
- {
- struct kmemleak_object *object;
- unsigned long flags;
- int i;
- if (!kmemleak_enabled || !log->ptr || IS_ERR(log->ptr))
- return;
-
- rcu_read_lock();
- object = create_object((unsigned long)log->ptr, log->size,
- log->min_count, GFP_ATOMIC);
- if (!object)
- goto out;
- spin_lock_irqsave(&object->lock, flags);
- for (i = 0; i < log->trace_len; i++)
- object->trace[i] = log->trace[i];
- object->trace_len = log->trace_len;
- spin_unlock_irqrestore(&object->lock, flags);
- out:
- rcu_read_unlock();
- }
- static void early_alloc_percpu(struct early_log *log)
- {
- unsigned int cpu;
- const void __percpu *ptr = log->ptr;
- for_each_possible_cpu(cpu) {
- log->ptr = per_cpu_ptr(ptr, cpu);
- early_alloc(log);
- }
- }
- void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
- gfp_t gfp)
- {
- pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);
- if (kmemleak_enabled && ptr && !IS_ERR(ptr))
- create_object((unsigned long)ptr, size, min_count, gfp);
- else if (kmemleak_early_log)
- log_early(KMEMLEAK_ALLOC, ptr, size, min_count);
- }
- EXPORT_SYMBOL_GPL(kmemleak_alloc);
- void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size,
- gfp_t gfp)
- {
- unsigned int cpu;
- pr_debug("%s(0x%p, %zu)\n", __func__, ptr, size);
-
- if (kmemleak_enabled && ptr && !IS_ERR(ptr))
- for_each_possible_cpu(cpu)
- create_object((unsigned long)per_cpu_ptr(ptr, cpu),
- size, 0, gfp);
- else if (kmemleak_early_log)
- log_early(KMEMLEAK_ALLOC_PERCPU, ptr, size, 0);
- }
- EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
- void __ref kmemleak_free(const void *ptr)
- {
- pr_debug("%s(0x%p)\n", __func__, ptr);
- if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
- delete_object_full((unsigned long)ptr);
- else if (kmemleak_early_log)
- log_early(KMEMLEAK_FREE, ptr, 0, 0);
- }
- EXPORT_SYMBOL_GPL(kmemleak_free);
- void __ref kmemleak_free_part(const void *ptr, size_t size)
- {
- pr_debug("%s(0x%p)\n", __func__, ptr);
- if (kmemleak_enabled && ptr && !IS_ERR(ptr))
- delete_object_part((unsigned long)ptr, size);
- else if (kmemleak_early_log)
- log_early(KMEMLEAK_FREE_PART, ptr, size, 0);
- }
- EXPORT_SYMBOL_GPL(kmemleak_free_part);
- void __ref kmemleak_free_percpu(const void __percpu *ptr)
- {
- unsigned int cpu;
- pr_debug("%s(0x%p)\n", __func__, ptr);
- if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
- for_each_possible_cpu(cpu)
- delete_object_full((unsigned long)per_cpu_ptr(ptr,
- cpu));
- else if (kmemleak_early_log)
- log_early(KMEMLEAK_FREE_PERCPU, ptr, 0, 0);
- }
- EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
- void __ref kmemleak_update_trace(const void *ptr)
- {
- struct kmemleak_object *object;
- unsigned long flags;
- pr_debug("%s(0x%p)\n", __func__, ptr);
- if (!kmemleak_enabled || IS_ERR_OR_NULL(ptr))
- return;
- object = find_and_get_object((unsigned long)ptr, 1);
- if (!object) {
- #ifdef DEBUG
- kmemleak_warn("Updating stack trace for unknown object at %p\n",
- ptr);
- #endif
- return;
- }
- spin_lock_irqsave(&object->lock, flags);
- object->trace_len = __save_stack_trace(object->trace);
- spin_unlock_irqrestore(&object->lock, flags);
- put_object(object);
- }
- EXPORT_SYMBOL(kmemleak_update_trace);
- void __ref kmemleak_not_leak(const void *ptr)
- {
- pr_debug("%s(0x%p)\n", __func__, ptr);
- if (kmemleak_enabled && ptr && !IS_ERR(ptr))
- make_gray_object((unsigned long)ptr);
- else if (kmemleak_early_log)
- log_early(KMEMLEAK_NOT_LEAK, ptr, 0, 0);
- }
- EXPORT_SYMBOL(kmemleak_not_leak);
- void __ref kmemleak_ignore(const void *ptr)
- {
- pr_debug("%s(0x%p)\n", __func__, ptr);
- if (kmemleak_enabled && ptr && !IS_ERR(ptr))
- make_black_object((unsigned long)ptr);
- else if (kmemleak_early_log)
- log_early(KMEMLEAK_IGNORE, ptr, 0, 0);
- }
- EXPORT_SYMBOL(kmemleak_ignore);
- void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
- {
- pr_debug("%s(0x%p)\n", __func__, ptr);
- if (kmemleak_enabled && ptr && size && !IS_ERR(ptr))
- add_scan_area((unsigned long)ptr, size, gfp);
- else if (kmemleak_early_log)
- log_early(KMEMLEAK_SCAN_AREA, ptr, size, 0);
- }
- EXPORT_SYMBOL(kmemleak_scan_area);
- void __ref kmemleak_no_scan(const void *ptr)
- {
- pr_debug("%s(0x%p)\n", __func__, ptr);
- if (kmemleak_enabled && ptr && !IS_ERR(ptr))
- object_no_scan((unsigned long)ptr);
- else if (kmemleak_early_log)
- log_early(KMEMLEAK_NO_SCAN, ptr, 0, 0);
- }
- EXPORT_SYMBOL(kmemleak_no_scan);
- void __ref kmemleak_alloc_phys(phys_addr_t phys, size_t size, int min_count,
- gfp_t gfp)
- {
- if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
- kmemleak_alloc(__va(phys), size, min_count, gfp);
- }
- EXPORT_SYMBOL(kmemleak_alloc_phys);
- void __ref kmemleak_free_part_phys(phys_addr_t phys, size_t size)
- {
- if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
- kmemleak_free_part(__va(phys), size);
- }
- EXPORT_SYMBOL(kmemleak_free_part_phys);
- void __ref kmemleak_not_leak_phys(phys_addr_t phys)
- {
- if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
- kmemleak_not_leak(__va(phys));
- }
- EXPORT_SYMBOL(kmemleak_not_leak_phys);
- void __ref kmemleak_ignore_phys(phys_addr_t phys)
- {
- if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
- kmemleak_ignore(__va(phys));
- }
- EXPORT_SYMBOL(kmemleak_ignore_phys);
- static bool update_checksum(struct kmemleak_object *object)
- {
- u32 old_csum = object->checksum;
- if (!kmemcheck_is_obj_initialized(object->pointer, object->size))
- return false;
- kasan_disable_current();
- object->checksum = crc32(0, (void *)object->pointer, object->size);
- kasan_enable_current();
- return object->checksum != old_csum;
- }
- static int scan_should_stop(void)
- {
- if (!kmemleak_enabled)
- return 1;
-
- if (current->mm)
- return signal_pending(current);
- else
- return kthread_should_stop();
- return 0;
- }
- static void scan_block(void *_start, void *_end,
- struct kmemleak_object *scanned)
- {
- unsigned long *ptr;
- unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
- unsigned long *end = _end - (BYTES_PER_POINTER - 1);
- unsigned long flags;
- read_lock_irqsave(&kmemleak_lock, flags);
- for (ptr = start; ptr < end; ptr++) {
- struct kmemleak_object *object;
- unsigned long pointer;
- if (scan_should_stop())
- break;
-
- if (!kmemcheck_is_obj_initialized((unsigned long)ptr,
- BYTES_PER_POINTER))
- continue;
- kasan_disable_current();
- pointer = *ptr;
- kasan_enable_current();
- if (pointer < min_addr || pointer >= max_addr)
- continue;
-
- object = lookup_object(pointer, 1);
- if (!object)
- continue;
- if (object == scanned)
-
- continue;
-
- spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
- if (!color_white(object)) {
-
- spin_unlock(&object->lock);
- continue;
- }
-
- object->count++;
- if (color_gray(object)) {
-
- WARN_ON(!get_object(object));
- list_add_tail(&object->gray_list, &gray_list);
- }
- spin_unlock(&object->lock);
- }
- read_unlock_irqrestore(&kmemleak_lock, flags);
- }
- static void scan_large_block(void *start, void *end)
- {
- void *next;
- while (start < end) {
- next = min(start + MAX_SCAN_SIZE, end);
- scan_block(start, next, NULL);
- start = next;
- cond_resched();
- }
- }
- static void scan_object(struct kmemleak_object *object)
- {
- struct kmemleak_scan_area *area;
- unsigned long flags;
-
- spin_lock_irqsave(&object->lock, flags);
- if (object->flags & OBJECT_NO_SCAN)
- goto out;
- if (!(object->flags & OBJECT_ALLOCATED))
-
- goto out;
- if (hlist_empty(&object->area_list)) {
- void *start = (void *)object->pointer;
- void *end = (void *)(object->pointer + object->size);
- void *next;
- do {
- next = min(start + MAX_SCAN_SIZE, end);
- scan_block(start, next, object);
- start = next;
- if (start >= end)
- break;
- spin_unlock_irqrestore(&object->lock, flags);
- cond_resched();
- spin_lock_irqsave(&object->lock, flags);
- } while (object->flags & OBJECT_ALLOCATED);
- } else
- hlist_for_each_entry(area, &object->area_list, node)
- scan_block((void *)area->start,
- (void *)(area->start + area->size),
- object);
- out:
- spin_unlock_irqrestore(&object->lock, flags);
- }
- static void scan_gray_list(void)
- {
- struct kmemleak_object *object, *tmp;
-
- object = list_entry(gray_list.next, typeof(*object), gray_list);
- while (&object->gray_list != &gray_list) {
- cond_resched();
-
- if (!scan_should_stop())
- scan_object(object);
- tmp = list_entry(object->gray_list.next, typeof(*object),
- gray_list);
-
- list_del(&object->gray_list);
- put_object(object);
- object = tmp;
- }
- WARN_ON(!list_empty(&gray_list));
- }
- static void kmemleak_scan(void)
- {
- unsigned long flags;
- struct kmemleak_object *object;
- int i;
- int new_leaks = 0;
- jiffies_last_scan = jiffies;
-
- rcu_read_lock();
- list_for_each_entry_rcu(object, &object_list, object_list) {
- spin_lock_irqsave(&object->lock, flags);
- #ifdef DEBUG
-
- if (atomic_read(&object->use_count) > 1) {
- pr_debug("object->use_count = %d\n",
- atomic_read(&object->use_count));
- dump_object_info(object);
- }
- #endif
-
- object->count = 0;
- if (color_gray(object) && get_object(object))
- list_add_tail(&object->gray_list, &gray_list);
- spin_unlock_irqrestore(&object->lock, flags);
- }
- rcu_read_unlock();
-
- scan_large_block(_sdata, _edata);
- scan_large_block(__bss_start, __bss_stop);
- scan_large_block(__start_data_ro_after_init, __end_data_ro_after_init);
- #ifdef CONFIG_SMP
-
- for_each_possible_cpu(i)
- scan_large_block(__per_cpu_start + per_cpu_offset(i),
- __per_cpu_end + per_cpu_offset(i));
- #endif
-
- get_online_mems();
- for_each_online_node(i) {
- unsigned long start_pfn = node_start_pfn(i);
- unsigned long end_pfn = node_end_pfn(i);
- unsigned long pfn;
- for (pfn = start_pfn; pfn < end_pfn; pfn++) {
- struct page *page;
- if (!pfn_valid(pfn))
- continue;
- page = pfn_to_page(pfn);
-
- if (page_count(page) == 0)
- continue;
- scan_block(page, page + 1, NULL);
- if (!(pfn % (MAX_SCAN_SIZE / sizeof(*page))))
- cond_resched();
- }
- }
- put_online_mems();
-
- if (kmemleak_stack_scan) {
- struct task_struct *p, *g;
- read_lock(&tasklist_lock);
- do_each_thread(g, p) {
- void *stack = try_get_task_stack(p);
- if (stack) {
- scan_block(stack, stack + THREAD_SIZE, NULL);
- put_task_stack(p);
- }
- } while_each_thread(g, p);
- read_unlock(&tasklist_lock);
- }
-
- scan_gray_list();
-
- rcu_read_lock();
- list_for_each_entry_rcu(object, &object_list, object_list) {
- spin_lock_irqsave(&object->lock, flags);
- if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
- && update_checksum(object) && get_object(object)) {
-
- object->count = object->min_count;
- list_add_tail(&object->gray_list, &gray_list);
- }
- spin_unlock_irqrestore(&object->lock, flags);
- }
- rcu_read_unlock();
-
- scan_gray_list();
-
- if (scan_should_stop())
- return;
-
- rcu_read_lock();
- list_for_each_entry_rcu(object, &object_list, object_list) {
- spin_lock_irqsave(&object->lock, flags);
- if (unreferenced_object(object) &&
- !(object->flags & OBJECT_REPORTED)) {
- object->flags |= OBJECT_REPORTED;
- new_leaks++;
- }
- spin_unlock_irqrestore(&object->lock, flags);
- }
- rcu_read_unlock();
- if (new_leaks) {
- kmemleak_found_leaks = true;
- pr_info("%d new suspected memory leaks (see /sys/kernel/debug/kmemleak)\n",
- new_leaks);
- }
- }
- static int kmemleak_scan_thread(void *arg)
- {
- static int first_run = 1;
- pr_info("Automatic memory scanning thread started\n");
- set_user_nice(current, 10);
-
- if (first_run) {
- signed long timeout = msecs_to_jiffies(SECS_FIRST_SCAN * 1000);
- first_run = 0;
- while (timeout && !kthread_should_stop())
- timeout = schedule_timeout_interruptible(timeout);
- }
- while (!kthread_should_stop()) {
- signed long timeout = jiffies_scan_wait;
- mutex_lock(&scan_mutex);
- kmemleak_scan();
- mutex_unlock(&scan_mutex);
-
- while (timeout && !kthread_should_stop())
- timeout = schedule_timeout_interruptible(timeout);
- }
- pr_info("Automatic memory scanning thread ended\n");
- return 0;
- }
- static void start_scan_thread(void)
- {
- if (scan_thread)
- return;
- scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
- if (IS_ERR(scan_thread)) {
- pr_warn("Failed to create the scan thread\n");
- scan_thread = NULL;
- }
- }
- static void stop_scan_thread(void)
- {
- if (scan_thread) {
- kthread_stop(scan_thread);
- scan_thread = NULL;
- }
- }
- static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)
- {
- struct kmemleak_object *object;
- loff_t n = *pos;
- int err;
- err = mutex_lock_interruptible(&scan_mutex);
- if (err < 0)
- return ERR_PTR(err);
- rcu_read_lock();
- list_for_each_entry_rcu(object, &object_list, object_list) {
- if (n-- > 0)
- continue;
- if (get_object(object))
- goto out;
- }
- object = NULL;
- out:
- return object;
- }
- static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)
- {
- struct kmemleak_object *prev_obj = v;
- struct kmemleak_object *next_obj = NULL;
- struct kmemleak_object *obj = prev_obj;
- ++(*pos);
- list_for_each_entry_continue_rcu(obj, &object_list, object_list) {
- if (get_object(obj)) {
- next_obj = obj;
- break;
- }
- }
- put_object(prev_obj);
- return next_obj;
- }
- static void kmemleak_seq_stop(struct seq_file *seq, void *v)
- {
- if (!IS_ERR(v)) {
-
- rcu_read_unlock();
- mutex_unlock(&scan_mutex);
- if (v)
- put_object(v);
- }
- }
- static int kmemleak_seq_show(struct seq_file *seq, void *v)
- {
- struct kmemleak_object *object = v;
- unsigned long flags;
- spin_lock_irqsave(&object->lock, flags);
- if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
- print_unreferenced(seq, object);
- spin_unlock_irqrestore(&object->lock, flags);
- return 0;
- }
- static const struct seq_operations kmemleak_seq_ops = {
- .start = kmemleak_seq_start,
- .next = kmemleak_seq_next,
- .stop = kmemleak_seq_stop,
- .show = kmemleak_seq_show,
- };
- static int kmemleak_open(struct inode *inode, struct file *file)
- {
- return seq_open(file, &kmemleak_seq_ops);
- }
- static int dump_str_object_info(const char *str)
- {
- unsigned long flags;
- struct kmemleak_object *object;
- unsigned long addr;
- if (kstrtoul(str, 0, &addr))
- return -EINVAL;
- object = find_and_get_object(addr, 0);
- if (!object) {
- pr_info("Unknown object at 0x%08lx\n", addr);
- return -EINVAL;
- }
- spin_lock_irqsave(&object->lock, flags);
- dump_object_info(object);
- spin_unlock_irqrestore(&object->lock, flags);
- put_object(object);
- return 0;
- }
- static void kmemleak_clear(void)
- {
- struct kmemleak_object *object;
- unsigned long flags;
- rcu_read_lock();
- list_for_each_entry_rcu(object, &object_list, object_list) {
- spin_lock_irqsave(&object->lock, flags);
- if ((object->flags & OBJECT_REPORTED) &&
- unreferenced_object(object))
- __paint_it(object, KMEMLEAK_GREY);
- spin_unlock_irqrestore(&object->lock, flags);
- }
- rcu_read_unlock();
- kmemleak_found_leaks = false;
- }
- static void __kmemleak_do_cleanup(void);
- static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,
- size_t size, loff_t *ppos)
- {
- char buf[64];
- int buf_size;
- int ret;
- buf_size = min(size, (sizeof(buf) - 1));
- if (strncpy_from_user(buf, user_buf, buf_size) < 0)
- return -EFAULT;
- buf[buf_size] = 0;
- ret = mutex_lock_interruptible(&scan_mutex);
- if (ret < 0)
- return ret;
- if (strncmp(buf, "clear", 5) == 0) {
- if (kmemleak_enabled)
- kmemleak_clear();
- else
- __kmemleak_do_cleanup();
- goto out;
- }
- if (!kmemleak_enabled) {
- ret = -EBUSY;
- goto out;
- }
- if (strncmp(buf, "off", 3) == 0)
- kmemleak_disable();
- else if (strncmp(buf, "stack=on", 8) == 0)
- kmemleak_stack_scan = 1;
- else if (strncmp(buf, "stack=off", 9) == 0)
- kmemleak_stack_scan = 0;
- else if (strncmp(buf, "scan=on", 7) == 0)
- start_scan_thread();
- else if (strncmp(buf, "scan=off", 8) == 0)
- stop_scan_thread();
- else if (strncmp(buf, "scan=", 5) == 0) {
- unsigned long secs;
- ret = kstrtoul(buf + 5, 0, &secs);
- if (ret < 0)
- goto out;
- stop_scan_thread();
- if (secs) {
- jiffies_scan_wait = msecs_to_jiffies(secs * 1000);
- start_scan_thread();
- }
- } else if (strncmp(buf, "scan", 4) == 0)
- kmemleak_scan();
- else if (strncmp(buf, "dump=", 5) == 0)
- ret = dump_str_object_info(buf + 5);
- else
- ret = -EINVAL;
- out:
- mutex_unlock(&scan_mutex);
- if (ret < 0)
- return ret;
-
- *ppos += size;
- return size;
- }
- static const struct file_operations kmemleak_fops = {
- .owner = THIS_MODULE,
- .open = kmemleak_open,
- .read = seq_read,
- .write = kmemleak_write,
- .llseek = seq_lseek,
- .release = seq_release,
- };
- static void __kmemleak_do_cleanup(void)
- {
- struct kmemleak_object *object;
- rcu_read_lock();
- list_for_each_entry_rcu(object, &object_list, object_list)
- delete_object_full(object->pointer);
- rcu_read_unlock();
- }
- static void kmemleak_do_cleanup(struct work_struct *work)
- {
- stop_scan_thread();
- mutex_lock(&scan_mutex);
-
- kmemleak_free_enabled = 0;
- mutex_unlock(&scan_mutex);
- if (!kmemleak_found_leaks)
- __kmemleak_do_cleanup();
- else
- pr_info("Kmemleak disabled without freeing internal data. Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\".\n");
- }
- static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
- static void kmemleak_disable(void)
- {
-
- if (cmpxchg(&kmemleak_error, 0, 1))
- return;
-
- kmemleak_enabled = 0;
-
- if (kmemleak_initialized)
- schedule_work(&cleanup_work);
- else
- kmemleak_free_enabled = 0;
- pr_info("Kernel memory leak detector disabled\n");
- }
- static int kmemleak_boot_config(char *str)
- {
- if (!str)
- return -EINVAL;
- if (strcmp(str, "off") == 0)
- kmemleak_disable();
- else if (strcmp(str, "on") == 0)
- kmemleak_skip_disable = 1;
- else
- return -EINVAL;
- return 0;
- }
- early_param("kmemleak", kmemleak_boot_config);
- static void __init print_log_trace(struct early_log *log)
- {
- struct stack_trace trace;
- trace.nr_entries = log->trace_len;
- trace.entries = log->trace;
- pr_notice("Early log backtrace:\n");
- print_stack_trace(&trace, 2);
- }
- void __init kmemleak_init(void)
- {
- int i;
- unsigned long flags;
- #ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
- if (!kmemleak_skip_disable) {
- kmemleak_early_log = 0;
- kmemleak_disable();
- return;
- }
- #endif
- jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
- jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
- object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
- scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
- if (crt_early_log > ARRAY_SIZE(early_log))
- pr_warn("Early log buffer exceeded (%d), please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n",
- crt_early_log);
-
- local_irq_save(flags);
- kmemleak_early_log = 0;
- if (kmemleak_error) {
- local_irq_restore(flags);
- return;
- } else {
- kmemleak_enabled = 1;
- kmemleak_free_enabled = 1;
- }
- local_irq_restore(flags);
-
- for (i = 0; i < crt_early_log; i++) {
- struct early_log *log = &early_log[i];
- switch (log->op_type) {
- case KMEMLEAK_ALLOC:
- early_alloc(log);
- break;
- case KMEMLEAK_ALLOC_PERCPU:
- early_alloc_percpu(log);
- break;
- case KMEMLEAK_FREE:
- kmemleak_free(log->ptr);
- break;
- case KMEMLEAK_FREE_PART:
- kmemleak_free_part(log->ptr, log->size);
- break;
- case KMEMLEAK_FREE_PERCPU:
- kmemleak_free_percpu(log->ptr);
- break;
- case KMEMLEAK_NOT_LEAK:
- kmemleak_not_leak(log->ptr);
- break;
- case KMEMLEAK_IGNORE:
- kmemleak_ignore(log->ptr);
- break;
- case KMEMLEAK_SCAN_AREA:
- kmemleak_scan_area(log->ptr, log->size, GFP_KERNEL);
- break;
- case KMEMLEAK_NO_SCAN:
- kmemleak_no_scan(log->ptr);
- break;
- default:
- kmemleak_warn("Unknown early log operation: %d\n",
- log->op_type);
- }
- if (kmemleak_warning) {
- print_log_trace(log);
- kmemleak_warning = 0;
- }
- }
- }
- static int __init kmemleak_late_init(void)
- {
- struct dentry *dentry;
- kmemleak_initialized = 1;
- if (kmemleak_error) {
-
- schedule_work(&cleanup_work);
- return -ENOMEM;
- }
- dentry = debugfs_create_file("kmemleak", S_IRUGO, NULL, NULL,
- &kmemleak_fops);
- if (!dentry)
- pr_warn("Failed to create the debugfs kmemleak file\n");
- mutex_lock(&scan_mutex);
- start_scan_thread();
- mutex_unlock(&scan_mutex);
- pr_info("Kernel memory leak detector initialized\n");
- return 0;
- }
- late_initcall(kmemleak_late_init);
|