Index: linux-2.6.15-rc1/arch/arm/mm/init.c =================================================================== --- linux-2.6.15-rc1.orig/arch/arm/mm/init.c 2005-11-20 10:49:24.000000000 +0000 +++ linux-2.6.15-rc1/arch/arm/mm/init.c 2005-11-20 14:29:55.000000000 +0000 @@ -96,6 +96,12 @@ return pmd_off(pgd_offset_k(virt), virt); } +struct node_info { + unsigned int start; + unsigned int end; + int bootmap_pages; +}; + #define for_each_nodebank(iter,mi,no) \ for (iter = 0; iter < mi->nr_banks; iter++) \ if (mi->bank[iter].node == no) @@ -182,8 +188,9 @@ /* * Reserve the various regions of node 0 */ -static __init void reserve_node_zero(pg_data_t *pgdat) +static __init void reserve_node_zero(unsigned int bootmap_pfn, unsigned int bootmap_pages) { + pg_data_t *pgdat = NODE_DATA(0); unsigned long res_size = 0; /* @@ -204,6 +211,13 @@ PTRS_PER_PGD * sizeof(pgd_t)); /* + * And don't forget to reserve the allocator bitmap, + * which will be freed later. + */ + reserve_bootmem_node(pgdat, bootmap_pfn << PAGE_SHIFT, + bootmap_pages << PAGE_SHIFT); + + /* * Hmm... This should go elsewhere, but we really really need to * stop things allocating the low memory; ideally we need a better * implementation of GFP_DMA which does not assume that DMA-able @@ -235,100 +249,155 @@ void __init build_mem_type_table(void); void __init create_mapping(struct map_desc *md); -static unsigned long __init -bootmem_init_node(int node, int initrd_node, struct meminfo *mi) +/* + * Register all available RAM in this node with the bootmem allocator. + */ +static inline void free_bootmem_node_bank(int node, struct meminfo *mi) { - unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES]; - unsigned long start_pfn, end_pfn, boot_pfn; - unsigned int boot_pages; - pg_data_t *pgdat; - int i; + pg_data_t *pgdat = NODE_DATA(node); + int bank; + + for_each_nodebank(bank, mi, node) + free_bootmem_node(pgdat, mi->bank[bank].start, mi->bank[bank].size); +} - start_pfn = -1UL; - end_pfn = 0; +static void fibble2(unsigned int bootmem_pages, struct node_info *np, struct meminfo *mi, unsigned int bootmap_pfn, unsigned int map_pg, unsigned long memend_pfn, int initrd_node) +{ + int i, node; /* - * Calculate the pfn range, and map the memory banks for this node. + * Calculate the number of pages we require to + * store the bootmem bitmaps. */ - for_each_nodebank(i, mi, node) { - unsigned long start, end; - struct map_desc map; - - start = mi->bank[i].start >> PAGE_SHIFT; - end = (mi->bank[i].start + mi->bank[i].size) >> PAGE_SHIFT; - - if (start_pfn > start) - start_pfn = start; - if (end_pfn < end) - end_pfn = end; - - map.pfn = __phys_to_pfn(mi->bank[i].start); - map.virtual = __phys_to_virt(mi->bank[i].start); - map.length = mi->bank[i].size; - map.type = MT_MEMORY; + for_each_online_node(i) { + if (np[i].end == 0) + continue; - create_mapping(&map); + np[i].bootmap_pages = bootmem_bootmap_pages(np[i].end - + np[i].start); + bootmem_pages += np[i].bootmap_pages; } - /* - * If there is no memory in this node, ignore it. - */ - if (end_pfn == 0) - return end_pfn; + high_memory = __va(memend_pfn << PAGE_SHIFT); /* - * Allocate the bootmem bitmap page. + * This doesn't seem to be used by the Linux memory + * manager any more. If we can get rid of it, we + * also get rid of some of the stuff above as well. + * + * Note: max_low_pfn and max_pfn reflect the number + * of _pages_ in the system, not the maximum PFN. */ - boot_pages = bootmem_bootmap_pages(end_pfn - start_pfn); - boot_pfn = find_bootmap_pfn(node, mi, boot_pages); + max_low_pfn = memend_pfn - (PHYS_OFFSET >> PAGE_SHIFT); + max_pfn = memend_pfn - (PHYS_OFFSET >> PAGE_SHIFT); - /* - * Initialise the bootmem allocator for this node, handing the - * memory banks over to bootmem. - */ - node_set_online(node); - pgdat = NODE_DATA(node); - init_bootmem_node(pgdat, boot_pfn, start_pfn, end_pfn); + bootmap_pfn = find_bootmap_pfn(0, mi, bootmem_pages); + initrd_node = check_initrd(mi); - for_each_nodebank(i, mi, node) - free_bootmem_node(pgdat, mi->bank[i].start, mi->bank[i].size); + map_pg = bootmap_pfn; /* - * Reserve the bootmem bitmap for this node. + * Initialise the bootmem nodes. + * + * What we really want to do is: + * + * unmap_all_regions_except_kernel(); + * for_each_node_in_reverse_order(node) { + * map_node(node); + * allocate_bootmem_map(node); + * init_bootmem_node(node); + * free_bootmem_node(node); + * } + * + * but this is a 2.5-type change. For now, we just set + * the nodes up in reverse order. + * + * (we could also do with rolling bootmem_init and paging_init + * into one generic "memory_init" type function). */ - reserve_bootmem_node(pgdat, boot_pfn << PAGE_SHIFT, - boot_pages << PAGE_SHIFT); + np += num_online_nodes() - 1; + for (node = num_online_nodes() - 1; node >= 0; node--, np--) { + /* + * If there are no pages in this node, ignore it. + * Note that node 0 must always have some pages. + */ + if (np->end == 0 || !node_online(node)) { + if (node == 0) + BUG(); + continue; + } + + /* + * Initialise the bootmem allocator. + */ + init_bootmem_node(NODE_DATA(node), map_pg, np->start, np->end); + free_bootmem_node_bank(node, mi); + map_pg += np->bootmap_pages; + + /* + * If this is node 0, we need to reserve some areas ASAP - + * we may use bootmem on node 0 to setup the other nodes. + */ + if (node == 0) + reserve_node_zero(bootmap_pfn, bootmem_pages); + } #ifdef CONFIG_BLK_DEV_INITRD /* * If the initrd is in this node, reserve its memory. */ - if (node == initrd_node) { - reserve_bootmem_node(pgdat, phys_initrd_start, + if (phys_initrd_size && initrd_node >= 0) { + reserve_bootmem_node(NODE_DATA(initrd_node), phys_initrd_start, phys_initrd_size); initrd_start = __phys_to_virt(phys_initrd_start); initrd_end = initrd_start + phys_initrd_size; } #endif + BUG_ON(map_pg != bootmap_pfn + bootmem_pages); +} + +static void fibble(struct meminfo *mi) +{ + unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES]; + unsigned long start_pfn, end_pfn, boot_pfn; + unsigned int boot_pages; + + struct bootmem_data *bdata; + int node; + pg_data_t *pgdat; + int i; + /* - * Finally, reserve any node zero regions. + * initialise the zones within each node */ - if (node == 0) - reserve_node_zero(pgdat); + for_each_online_node(node) { /* - * initialise the zones within this node. + * Initialise the zone size information. */ - memset(zone_size, 0, sizeof(zone_size)); - memset(zhole_size, 0, sizeof(zhole_size)); + for (i = 0; i < MAX_NR_ZONES; i++) { + zone_size[i] = 0; + zhole_size[i] = 0; + } + + pgdat = NODE_DATA(node); + bdata = pgdat->bdata; /* - * The size of this node has already been determined. If we need - * to do anything fancy with the allocation of this memory to the - * zones, now is the time to do it. + * The size of this node has already been determined. + * If we need to do anything fancy with the allocation + * of this memory to the zones, now is the time to do + * it. */ - zone_size[0] = end_pfn - start_pfn; + zone_size[0] = bdata->node_low_pfn - + (bdata->node_boot_start >> PAGE_SHIFT); + + /* + * If this zone has zero size, skip it. + */ + if (!zone_size[0]) + continue; /* * For each bank in this node, calculate the size of the holes. @@ -344,16 +413,20 @@ */ arch_adjust_zones(node, zone_size, zhole_size); - free_area_init_node(node, pgdat, zone_size, start_pfn, zhole_size); - - return end_pfn; + free_area_init_node(node, pgdat, zone_size, + bdata->node_boot_start >> PAGE_SHIFT, zhole_size); + } } + static void __init bootmem_init(struct meminfo *mi) { unsigned long addr, memend_pfn = 0; int node, initrd_node, i; + struct node_info node_info[MAX_NUMNODES], *np = node_info; + unsigned int bootmem_pages = 0, bootmap_pfn, map_pg; + /* * Invalidate the node number for empty or invalid memory banks */ @@ -362,58 +435,49 @@ mi->bank[i].node = -1; memcpy(&meminfo, mi, sizeof(meminfo)); + + for (i = 0; i < MAX_NUMNODES; i++) { + np[i].start = -1U; + np[i].end = 0; + np[i].bootmap_pages = 0; + } - /* - * Clear out all the mappings below the kernel image. - */ - for (addr = 0; addr < MODULE_START; addr += PGDIR_SIZE) - pmd_clear(pmd_off_k(addr)); -#ifdef CONFIG_XIP_KERNEL - /* The XIP kernel is mapped in the module area -- skip over it */ - addr = ((unsigned long)&_etext + PGDIR_SIZE - 1) & PGDIR_MASK; -#endif - for ( ; addr < PAGE_OFFSET; addr += PGDIR_SIZE) - pmd_clear(pmd_off_k(addr)); - - /* - * Clear out all the kernel space mappings, except for the first - * memory bank, up to the end of the vmalloc region. - */ - for (addr = __phys_to_virt(mi->bank[0].start + mi->bank[0].size); - addr < VMALLOC_END; addr += PGDIR_SIZE) - pmd_clear(pmd_off_k(addr)); + for (i = 0; i < mi->nr_banks; i++) { + unsigned long start, end; + int node; - /* - * Locate which node contains the ramdisk image, if any. - */ - initrd_node = check_initrd(mi); + if (mi->bank[i].node == -1) + continue; - /* - * Run through each node initialising the bootmem allocator. - */ - for_each_node(node) { - unsigned long end_pfn; + node = mi->bank[i].node; - end_pfn = bootmem_init_node(node, initrd_node, mi); + /* + * Make sure we haven't exceeded the maximum number of nodes + * that we have in this configuration. If we have, we're in + * trouble. (maybe we ought to limit, instead of bugging?) + */ + if (node >= MAX_NUMNODES) + BUG(); + node_set_online(node); /* - * Remember the highest memory PFN. + * Get the start and end pfns for this bank */ - if (end_pfn > memend_pfn) - memend_pfn = end_pfn; - } + start = (mi->bank[i].start) >> PAGE_SHIFT; + end = (mi->bank[i].start + mi->bank[i].size) >> PAGE_SHIFT; - high_memory = __va(memend_pfn << PAGE_SHIFT); + if (np[node].start > start) + np[node].start = start; - /* - * This doesn't seem to be used by the Linux memory manager any - * more, but is used by ll_rw_block. If we can get rid of it, we - * also get rid of some of the stuff above as well. - * - * Note: max_low_pfn and max_pfn reflect the number of _pages_ in - * the system, not the maximum PFN. - */ - max_pfn = max_low_pfn = memend_pfn - PHYS_PFN_OFFSET; + if (np[node].end < end) + np[node].end = end; + + if (memend_pfn < end) + memend_pfn = end; + } + + fibble2(bootmem_pages, np, mi, bootmap_pfn, map_pg, memend_pfn, initrd_node); + fibble(mi); } /* @@ -422,11 +486,12 @@ * This means you have to be careful how you debug this function, or any * called function. (Do it by code inspection!) */ -static void __init devicemaps_init(struct machine_desc *mdesc) +static void __init devicemaps_init(struct machine_desc *mdesc, struct meminfo *mi) { struct map_desc map; unsigned long addr; void *vectors; + int i; for (addr = VMALLOC_END; addr; addr += PGDIR_SIZE) pmd_clear(pmd_off_k(addr)); @@ -467,6 +532,17 @@ vectors = alloc_bootmem_low_pages(PAGE_SIZE); BUG_ON(!vectors); + for (i = 0; i < mi->nr_banks; i++) { + if (mi->bank[i].size == 0) + continue; + + map.pfn = __phys_to_pfn(mi->bank[i].start); + map.virtual = __phys_to_virt(mi->bank[i].start); + map.length = mi->bank[i].size; + map.type = MT_MEMORY; + create_mapping(&map); + } + /* * Create a mapping for the machine vectors at the high-vectors * location (0xffff0000). If we aren't using high-vectors, also @@ -496,6 +572,7 @@ * draining. After this point, we can start to touch devices * again. */ + flush_cache_all(); local_flush_tlb_all(); } @@ -509,7 +586,7 @@ build_mem_type_table(); bootmem_init(mi); - devicemaps_init(mdesc); + devicemaps_init(mdesc, mi); top_pmd = pmd_off_k(0xffff0000); @@ -517,6 +594,7 @@ * allocate the zero page. Note that we count on this going ok. */ zero_page = alloc_bootmem_low_pages(PAGE_SIZE); + memzero(zero_page, PAGE_SIZE); empty_zero_page = virt_to_page(zero_page); flush_dcache_page(empty_zero_page); @@ -579,7 +657,8 @@ * information on the command line. */ for_each_nodebank(i, mi, node) { - bank_start = mi->bank[i].start >> PAGE_SHIFT; + + bank_start = __phys_to_pfn(mi->bank[i].start) >> PAGE_SHIFT; if (bank_start < prev_bank_end) { printk(KERN_ERR "MEM: unordered memory banks. " "Not freeing memmap.\n"); @@ -593,7 +672,7 @@ if (prev_bank_end && prev_bank_end != bank_start) free_memmap(node, prev_bank_end, bank_start); - prev_bank_end = (mi->bank[i].start + + prev_bank_end = (__phys_to_pfn(mi->bank[i].start) + mi->bank[i].size) >> PAGE_SHIFT; } }