@@ -2021,7 +2021,14 @@ generic_make_request_checks(struct bio *bio)
20212021 */
20222022blk_qc_t generic_make_request (struct bio * bio )
20232023{
2024- struct bio_list bio_list_on_stack ;
2024+ /*
2025+ * bio_list_on_stack[0] contains bios submitted by the current
2026+ * make_request_fn.
2027+ * bio_list_on_stack[1] contains bios that were submitted before
2028+ * the current make_request_fn, but that haven't been processed
2029+ * yet.
2030+ */
2031+ struct bio_list bio_list_on_stack [2 ];
20252032 blk_qc_t ret = BLK_QC_T_NONE ;
20262033
20272034 if (!generic_make_request_checks (bio ))
@@ -2038,7 +2045,7 @@ blk_qc_t generic_make_request(struct bio *bio)
20382045 * should be added at the tail
20392046 */
20402047 if (current -> bio_list ) {
2041- bio_list_add (current -> bio_list , bio );
2048+ bio_list_add (& current -> bio_list [ 0 ] , bio );
20422049 goto out ;
20432050 }
20442051
@@ -2057,24 +2064,39 @@ blk_qc_t generic_make_request(struct bio *bio)
20572064 * bio_list, and call into ->make_request() again.
20582065 */
20592066 BUG_ON (bio -> bi_next );
2060- bio_list_init (& bio_list_on_stack );
2061- current -> bio_list = & bio_list_on_stack ;
2067+ bio_list_init (& bio_list_on_stack [ 0 ] );
2068+ current -> bio_list = bio_list_on_stack ;
20622069 do {
20632070 struct request_queue * q = bdev_get_queue (bio -> bi_bdev );
20642071
20652072 if (likely (blk_queue_enter (q , __GFP_DIRECT_RECLAIM ) == 0 )) {
2073+ struct bio_list lower , same ;
2074+
2075+ /* Create a fresh bio_list for all subordinate requests */
2076+ bio_list_on_stack [1 ] = bio_list_on_stack [0 ];
2077+ bio_list_init (& bio_list_on_stack [0 ]);
20662078
20672079 ret = q -> make_request_fn (q , bio );
20682080
20692081 blk_queue_exit (q );
2070-
2071- bio = bio_list_pop (current -> bio_list );
2082+ /* sort new bios into those for a lower level
2083+ * and those for the same level
2084+ */
2085+ bio_list_init (& lower );
2086+ bio_list_init (& same );
2087+ while ((bio = bio_list_pop (& bio_list_on_stack [0 ])) != NULL )
2088+ if (q == bdev_get_queue (bio -> bi_bdev ))
2089+ bio_list_add (& same , bio );
2090+ else
2091+ bio_list_add (& lower , bio );
2092+ /* now assemble so we handle the lowest level first */
2093+ bio_list_merge (& bio_list_on_stack [0 ], & lower );
2094+ bio_list_merge (& bio_list_on_stack [0 ], & same );
2095+ bio_list_merge (& bio_list_on_stack [0 ], & bio_list_on_stack [1 ]);
20722096 } else {
2073- struct bio * bio_next = bio_list_pop (current -> bio_list );
2074-
20752097 bio_io_error (bio );
2076- bio = bio_next ;
20772098 }
2099+ bio = bio_list_pop (& bio_list_on_stack [0 ]);
20782100 } while (bio );
20792101 current -> bio_list = NULL ; /* deactivate */
20802102
0 commit comments