| /linux-master/include/linux/ |
| H A D | zswap.h | 8 struct lruvec;
36 void zswap_lruvec_state_init(struct lruvec *lruvec);
60 static inline void zswap_lruvec_state_init(struct lruvec *lruvec) {} argument
|
| H A D | mm_inline.h | 38 static __always_inline void __update_lru_size(struct lruvec *lruvec, argument
42 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
44 lockdep_assert_held(&lruvec->lru_lock);
47 __mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages);
52 static __always_inline void update_lru_size(struct lruvec *lruvec, argument
56 __update_lru_size(lruvec, lru, zid, nr_pages);
58 mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages);
165 static inline bool lru_gen_is_active(struct lruvec *lruve argument
175 lru_gen_update_size(struct lruvec *lruvec, struct folio *folio, int old_gen, int new_gen) argument
221 lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) argument
272 lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) argument
306 lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) argument
311 lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) argument
319 lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) argument
333 lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) argument
347 lruvec_del_folio(struct lruvec *lruvec, struct folio *folio) argument
[all...] |
| H A D | memcontrol.h | 99 struct lruvec lruvec; member in struct:mem_cgroup_per_node
730 * @memcg: memcg of the wanted lruvec
734 * @pgdat combination. This can be the node lruvec, if the memory
737 static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
741 struct lruvec *lruvec; local
744 lruvec = &pgdat->__lruvec;
752 lruvec = &mz->lruvec;
793 lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio) argument
874 lruvec_memcg(struct lruvec *lruvec) argument
932 mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx) argument
1284 lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio) argument
1405 lruvec_memcg(struct lruvec *lruvec) argument
1416 mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx) argument
1516 lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) argument
1522 lruvec_page_state_local(struct lruvec *lruvec, enum node_stat_item idx) argument
1610 parent_lruvec(struct lruvec *lruvec) argument
1623 unlock_page_lruvec(struct lruvec *lruvec) argument
1628 unlock_page_lruvec_irq(struct lruvec *lruvec) argument
1633 unlock_page_lruvec_irqrestore(struct lruvec *lruvec, unsigned long flags) argument
1640 folio_matches_lruvec(struct folio *folio, struct lruvec *lruvec) argument
[all...] |
| H A D | vmstat.h | 538 void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
541 static inline void mod_lruvec_state(struct lruvec *lruvec, argument
547 __mod_lruvec_state(lruvec, idx, val);
572 static inline void __mod_lruvec_state(struct lruvec *lruvec, argument
575 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
578 static inline void mod_lruvec_state(struct lruvec *lruvec, argument
[all...] |
| H A D | mmzone.h | 255 * Per-memcg and per-lruvec counters track memory, consumed
313 * An lruvec has many dirty pages backed by a congested BDI:
319 * Essentially, kswapd can unthrottle an lruvec throttled by cgroup
386 struct lruvec;
486 /* the lruvec under reclaim */
487 struct lruvec *lruvec; member in struct:lru_gen_mm_walk
559 void lru_gen_init_lruvec(struct lruvec *lruvec);
575 static inline void lru_gen_init_lruvec(struct lruvec *lruve argument
609 struct lruvec { struct
1458 lruvec_pgdat(struct lruvec *lruvec) argument
[all...] |
| H A D | swap.h | 358 void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);
371 void lru_note_cost(struct lruvec *lruvec, bool file,
|
| /linux-master/mm/ |
| H A D | swap.c | 77 static void __page_cache_release(struct folio *folio, struct lruvec **lruvecp,
107 struct lruvec *lruvec = NULL; local
110 __page_cache_release(folio, &lruvec, &flags);
111 if (lruvec)
112 unlock_page_lruvec_irqrestore(lruvec, flags);
164 typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio);
166 static void lru_add_fn(struct lruvec *lruvec, struc argument
209 struct lruvec *lruvec = NULL; local
239 lru_move_tail_fn(struct lruvec *lruvec, struct folio *folio) argument
271 lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_io, unsigned int nr_rotated) argument
329 folio_activate_fn(struct lruvec *lruvec, struct folio *folio) argument
375 struct lruvec *lruvec; local
566 lru_deactivate_file_fn(struct lruvec *lruvec, struct folio *folio) argument
607 lru_deactivate_fn(struct lruvec *lruvec, struct folio *folio) argument
623 lru_lazyfree_fn(struct lruvec *lruvec, struct folio *folio) argument
971 struct lruvec *lruvec = NULL; local
[all...] |
| H A D | mmzone.c | 75 void lruvec_init(struct lruvec *lruvec) argument
79 memset(lruvec, 0, sizeof(struct lruvec));
80 spin_lock_init(&lruvec->lru_lock);
81 zswap_lruvec_state_init(lruvec);
84 INIT_LIST_HEAD(&lruvec->lists[lru]);
91 list_del(&lruvec->lists[LRU_UNEVICTABLE]);
93 lru_gen_init_lruvec(lruvec);
|
| H A D | workingset.c | 192 * entry, and after storing the identifier for the lruvec there might
237 struct lruvec *lruvec; local
248 lruvec = mem_cgroup_lruvec(memcg, pgdat);
249 lrugen = &lruvec->lrugen;
261 * Fills in @lruvec, @token, @workingset with the values unpacked from shadow.
263 static bool lru_gen_test_recent(void *shadow, bool file, struct lruvec **lruvec, argument
274 *lruvec = mem_cgroup_lruvec(memcg, pgdat);
276 min_seq = READ_ONCE((*lruvec)
286 struct lruvec *lruvec; local
334 lru_gen_test_recent(void *shadow, bool file, struct lruvec **lruvec, unsigned long *token, bool *workingset) argument
356 workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages) argument
386 struct lruvec *lruvec; local
535 struct lruvec *lruvec; local
682 struct lruvec *lruvec; local
[all...] |
| H A D | vmscan.c | 362 * @lruvec: lru vector
366 static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, argument
373 struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
379 size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
1576 static __always_inline void update_lru_sizes(struct lruvec *lruvec, argument
1585 update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1610 * Isolating page from the lruvec to fill in @dst list by nr_to_scan times.
1612 * lruvec
1630 isolate_lru_folios(unsigned long nr_to_scan, struct lruvec *lruvec, struct list_head *dst, unsigned long *nr_scanned, struct scan_control *sc, enum lru_list lru) argument
1753 struct lruvec *lruvec; local
1814 move_folios_to_lru(struct lruvec *lruvec, struct list_head *list) argument
1898 shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, struct scan_control *sc, enum lru_list lru) argument
2016 shrink_active_list(unsigned long nr_to_scan, struct lruvec *lruvec, struct scan_control *sc, enum lru_list lru) argument
2168 shrink_list(enum lru_list lru, unsigned long nr_to_scan, struct lruvec *lruvec, struct scan_control *sc) argument
2210 inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru) argument
2350 get_scan_count(struct lruvec *lruvec, struct scan_control *sc, unsigned long *nr) argument
2611 struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; local
2625 get_swappiness(struct lruvec *lruvec, struct scan_control *sc) argument
2640 get_nr_gens(struct lruvec *lruvec, int type) argument
2645 seq_is_valid(struct lruvec *lruvec) argument
2769 get_mm_state(struct lruvec *lruvec) argument
2806 struct lruvec *lruvec = get_lruvec(memcg, nid); local
2836 struct lruvec *lruvec = get_lruvec(memcg, nid); local
2895 get_mm_state(struct lruvec *lruvec) argument
2911 struct lruvec *lruvec = walk->lruvec; local
2937 struct lruvec *lruvec = walk->lruvec; local
2996 iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long seq) argument
3048 read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain, struct ctrl_pos *pos) argument
3063 reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) argument
3140 folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) argument
3188 struct lruvec *lruvec = walk->lruvec; local
3629 struct lruvec *lruvec = walk->lruvec; local
3698 inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) argument
3734 try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) argument
3777 inc_max_seq(struct lruvec *lruvec, unsigned long seq, bool can_swap, bool force_scan) argument
3845 try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq, bool can_swap, bool force_scan) argument
3903 lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc) argument
3928 lruvec_is_reclaimable(struct lruvec *lruvec, struct scan_control *sc, unsigned long min_ttl) argument
3967 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); local
4018 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); local
4119 lru_gen_rotate_memcg(struct lruvec *lruvec, int op) argument
4175 struct lruvec *lruvec = get_lruvec(memcg, nid); local
4197 struct lruvec *lruvec = get_lruvec(memcg, nid); local
4210 struct lruvec *lruvec = get_lruvec(memcg, nid); local
4231 struct lruvec *lruvec = get_lruvec(memcg, nid); local
4244 sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc, int tier_idx) argument
4313 isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) argument
4347 scan_folios(struct lruvec *lruvec, struct scan_control *sc, int type, int tier, struct list_head *list) argument
4428 get_tier_idx(struct lruvec *lruvec, int type) argument
4448 get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx) argument
4476 isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, int *type_scanned, struct list_head *list) argument
4523 evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness) argument
4616 should_run_aging(struct lruvec *lruvec, unsigned long max_seq, bool can_swap, unsigned long *nr_to_scan) argument
4681 get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, bool can_swap) argument
4705 should_abort_scan(struct lruvec *lruvec, struct scan_control *sc) argument
4736 try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) argument
4767 shrink_one(struct lruvec *lruvec, struct scan_control *sc) argument
4815 struct lruvec *lruvec; local
4879 lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) argument
4968 state_is_valid(struct lruvec *lruvec) argument
4991 fill_evictable(struct lruvec *lruvec) argument
5022 drain_evictable(struct lruvec *lruvec) argument
5075 struct lruvec *lruvec = get_lruvec(memcg, nid); local
5238 lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq, unsigned long seq) argument
5298 struct lruvec *lruvec = v; local
5357 run_aging(struct lruvec *lruvec, unsigned long seq, bool can_swap, bool force_scan) argument
5377 run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, int swappiness, unsigned long nr_to_reclaim) argument
5408 struct lruvec *lruvec; local
5561 lru_gen_init_lruvec(struct lruvec *lruvec) argument
5603 struct lruvec *lruvec = get_lruvec(memcg, nid); local
5645 lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) argument
5657 shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) argument
5852 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); local
6256 struct lruvec *lruvec; local
6481 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); local
6558 struct lruvec *lruvec; local
6647 struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat); local
7517 struct lruvec *lruvec = NULL; local
[all...] |
| H A D | mlock.c | 61 static struct lruvec *__mlock_folio(struct folio *folio, struct lruvec *lruvec) argument
65 return lruvec;
67 lruvec = folio_lruvec_relock_irq(folio, lruvec);
76 lruvec_del_folio(lruvec, folio);
78 lruvec_add_folio(lruvec, folio);
92 lruvec_del_folio(lruvec, folio);
96 lruvec_add_folio(lruvec, foli
103 __mlock_new_folio(struct folio *folio, struct lruvec *lruvec) argument
122 __munlock_folio(struct folio *folio, struct lruvec *lruvec) argument
188 struct lruvec *lruvec = NULL; local
[all...] |
| H A D | memcontrol.c | 663 unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) argument
670 return node_page_state(lruvec_pgdat(lruvec), idx);
676 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
685 unsigned long lruvec_page_state_local(struct lruvec *lruvec, argument
693 return node_page_state(lruvec_pgdat(lruvec), idx);
699 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
990 __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, int val) argument
1043 __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, int val) argument
1059 struct lruvec *lruvec; local
1080 struct lruvec *lruvec; local
1501 lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio) argument
1531 struct lruvec *lruvec = folio_lruvec(folio); local
1554 struct lruvec *lruvec = folio_lruvec(folio); local
1579 struct lruvec *lruvec = folio_lruvec(folio); local
1597 mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, int zid, int nr_pages) argument
3148 struct lruvec *lruvec; local
4379 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); local
7114 lruvec_page_state_output(struct lruvec *lruvec, int item) argument
7137 struct lruvec *lruvec; local
[all...] |
| H A D | zswap.c | 717 struct lruvec *lruvec; local
738 lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
739 nr_zswap_protected = &lruvec->zswap_lruvec_state.nr_zswap_protected;
763 void zswap_lruvec_state_init(struct lruvec *lruvec) argument
765 atomic_long_set(&lruvec->zswap_lruvec_state.nr_zswap_protected, 0);
770 struct lruvec *lruvec; local
773 lruvec
1175 struct lruvec *lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(sc->nid)); local
1217 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(sc->nid)); local
[all...] |
| H A D | compaction.c | 896 struct lruvec *lruvec; local
898 struct lruvec *locked = NULL;
1213 lruvec = folio_lruvec(folio);
1216 if (lruvec != locked) {
1220 compact_lock_irqsave(&lruvec->lru_lock, &flags, cc);
1221 locked = lruvec;
1223 lruvec_memcg_debug(lruvec, folio);
1257 lruvec_del_folio(lruvec, folio);
|
| H A D | huge_memory.c | 2705 struct lruvec *lruvec, struct list_head *list)
2709 lockdep_assert_held(&lruvec->lru_lock);
2728 struct lruvec *lruvec, struct list_head *list,
2821 lru_add_page_tail(head, page_tail, lruvec, list);
2829 struct lruvec *lruvec; local
2847 lruvec = folio_lruvec_lock(folio);
2852 __split_huge_page_tail(folio, i, lruvec, lis
2704 lru_add_page_tail(struct page *head, struct page *tail, struct lruvec *lruvec, struct list_head *list) argument
2727 __split_huge_page_tail(struct folio *folio, int tail, struct lruvec *lruvec, struct list_head *list, unsigned int new_order) argument
[all...] |
| H A D | migrate.c | 489 struct lruvec *old_lruvec, *new_lruvec;
|