1 /* 2 * Copyright (C) 2009 Michael Brown <mbrown (at) fensystems.co.uk>. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License as 6 * published by the Free Software Foundation; either version 2 of the 7 * License, or any later version. 8 * 9 * This program is distributed in the hope that it will be useful, but 10 * WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 * General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 17 */ 18 19 FILE_LICENCE ( GPL2_OR_LATER ); 20 21 #include <stdint.h> 22 #include <stdlib.h> 23 #include <string.h> 24 #include <byteswap.h> 25 #include <errno.h> 26 #include <gpxe/infiniband.h> 27 #include <gpxe/ib_mi.h> 28 #include <gpxe/ib_pathrec.h> 29 30 /** @file 31 * 32 * Infiniband path lookups 33 * 34 */ 35 36 /** 37 * Handle path transaction completion 38 * 39 * @v ibdev Infiniband device 40 * @v mi Management interface 41 * @v madx Management transaction 42 * @v rc Status code 43 * @v mad Received MAD (or NULL on error) 44 * @v av Source address vector (or NULL on error) 45 */ 46 static void ib_path_complete ( struct ib_device *ibdev, 47 struct ib_mad_interface *mi, 48 struct ib_mad_transaction *madx, 49 int rc, union ib_mad *mad, 50 struct ib_address_vector *av __unused ) { 51 struct ib_path *path = ib_madx_get_ownerdata ( madx ); 52 struct ib_gid *dgid = &path->av.gid; 53 struct ib_path_record *pathrec = &mad->sa.sa_data.path_record; 54 55 /* Report failures */ 56 if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) )) 57 rc = -ENETUNREACH; 58 if ( rc != 0 ) { 59 DBGC ( ibdev, "IBDEV %p path lookup for %08x:%08x:%08x:%08x " 60 "failed: %s\n", ibdev, htonl ( dgid->u.dwords[0] ), 61 htonl ( dgid->u.dwords[1] ), 62 htonl ( dgid->u.dwords[2] ), 63 htonl ( dgid->u.dwords[3] ), strerror ( rc ) ); 64 goto out; 65 } 66 67 /* Extract values from MAD */ 68 path->av.lid = ntohs ( pathrec->dlid ); 69 path->av.sl = ( pathrec->reserved__sl & 0x0f ); 70 path->av.rate = ( pathrec->rate_selector__rate & 0x3f ); 71 DBGC ( ibdev, "IBDEV %p path to %08x:%08x:%08x:%08x is %04x sl %d " 72 "rate %d\n", ibdev, htonl ( dgid->u.dwords[0] ), 73 htonl ( dgid->u.dwords[1] ), htonl ( dgid->u.dwords[2] ), 74 htonl ( dgid->u.dwords[3] ), path->av.lid, path->av.sl, 75 path->av.rate ); 76 77 out: 78 /* Destroy the completed transaction */ 79 ib_destroy_madx ( ibdev, mi, madx ); 80 path->madx = NULL; 81 82 /* Hand off to upper completion handler */ 83 path->op->complete ( ibdev, path, rc, &path->av ); 84 } 85 86 /** Path transaction completion operations */ 87 static struct ib_mad_transaction_operations ib_path_op = { 88 .complete = ib_path_complete, 89 }; 90 91 /** 92 * Create path 93 * 94 * @v ibdev Infiniband device 95 * @v av Address vector to complete 96 * @v op Path operations 97 * @ret path Path 98 */ 99 struct ib_path * 100 ib_create_path ( struct ib_device *ibdev, struct ib_address_vector *av, 101 struct ib_path_operations *op ) { 102 struct ib_path *path; 103 union ib_mad mad; 104 struct ib_mad_sa *sa = &mad.sa; 105 106 /* Allocate and initialise structure */ 107 path = zalloc ( sizeof ( *path ) ); 108 if ( ! path ) 109 goto err_alloc_path; 110 path->ibdev = ibdev; 111 memcpy ( &path->av, av, sizeof ( path->av ) ); 112 path->op = op; 113 114 /* Construct path request */ 115 memset ( sa, 0, sizeof ( *sa ) ); 116 sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM; 117 sa->mad_hdr.class_version = IB_SA_CLASS_VERSION; 118 sa->mad_hdr.method = IB_MGMT_METHOD_GET; 119 sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_PATH_REC ); 120 sa->sa_hdr.comp_mask[1] = 121 htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID ); 122 memcpy ( &sa->sa_data.path_record.dgid, &path->av.gid, 123 sizeof ( sa->sa_data.path_record.dgid ) ); 124 memcpy ( &sa->sa_data.path_record.sgid, &ibdev->gid, 125 sizeof ( sa->sa_data.path_record.sgid ) ); 126 127 /* Create management transaction */ 128 path->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL, 129 &ib_path_op ); 130 if ( ! path->madx ) 131 goto err_create_madx; 132 ib_madx_set_ownerdata ( path->madx, path ); 133 134 return path; 135 136 ib_destroy_madx ( ibdev, ibdev->gsi, path->madx ); 137 err_create_madx: 138 free ( path ); 139 err_alloc_path: 140 return NULL; 141 } 142 143 /** 144 * Destroy path 145 * 146 * @v ibdev Infiniband device 147 * @v path Path 148 */ 149 void ib_destroy_path ( struct ib_device *ibdev, struct ib_path *path ) { 150 151 if ( path->madx ) 152 ib_destroy_madx ( ibdev, ibdev->gsi, path->madx ); 153 free ( path ); 154 } 155 156 /** Number of path cache entries 157 * 158 * Must be a power of two. 159 */ 160 #define IB_NUM_CACHED_PATHS 4 161 162 /** A cached path */ 163 struct ib_cached_path { 164 /** Path */ 165 struct ib_path *path; 166 }; 167 168 /** Path cache */ 169 static struct ib_cached_path ib_path_cache[IB_NUM_CACHED_PATHS]; 170 171 /** Oldest path cache entry index */ 172 static unsigned int ib_path_cache_idx; 173 174 /** 175 * Find path cache entry 176 * 177 * @v ibdev Infiniband device 178 * @v dgid Destination GID 179 * @ret path Path cache entry, or NULL 180 */ 181 static struct ib_cached_path * 182 ib_find_path_cache_entry ( struct ib_device *ibdev, struct ib_gid *dgid ) { 183 struct ib_cached_path *cached; 184 unsigned int i; 185 186 for ( i = 0 ; i < IB_NUM_CACHED_PATHS ; i++ ) { 187 cached = &ib_path_cache[i]; 188 if ( ! cached->path ) 189 continue; 190 if ( cached->path->ibdev != ibdev ) 191 continue; 192 if ( memcmp ( &cached->path->av.gid, dgid, 193 sizeof ( cached->path->av.gid ) ) != 0 ) 194 continue; 195 return cached; 196 } 197 198 return NULL; 199 } 200 201 /** 202 * Handle cached path transaction completion 203 * 204 * @v ibdev Infiniband device 205 * @v path Path 206 * @v rc Status code 207 * @v av Address vector, or NULL on error 208 */ 209 static void ib_cached_path_complete ( struct ib_device *ibdev, 210 struct ib_path *path, int rc, 211 struct ib_address_vector *av __unused ) { 212 struct ib_cached_path *cached = ib_path_get_ownerdata ( path ); 213 214 /* If the transaction failed, erase the cache entry */ 215 if ( rc != 0 ) { 216 /* Destroy the old cache entry */ 217 ib_destroy_path ( ibdev, path ); 218 memset ( cached, 0, sizeof ( *cached ) ); 219 return; 220 } 221 222 /* Do not destroy the completed transaction; we still need to 223 * refer to the resolved path. 224 */ 225 } 226 227 /** Cached path transaction completion operations */ 228 static struct ib_path_operations ib_cached_path_op = { 229 .complete = ib_cached_path_complete, 230 }; 231 232 /** 233 * Resolve path 234 * 235 * @v ibdev Infiniband device 236 * @v av Address vector to complete 237 * @ret rc Return status code 238 * 239 * This provides a non-transactional way to resolve a path, via a 240 * cache similar to ARP. 241 */ 242 int ib_resolve_path ( struct ib_device *ibdev, struct ib_address_vector *av ) { 243 struct ib_gid *gid = &av->gid; 244 struct ib_cached_path *cached; 245 unsigned int cache_idx; 246 247 /* Sanity check */ 248 if ( ! av->gid_present ) { 249 DBGC ( ibdev, "IBDEV %p attempt to look up path " 250 "without GID\n", ibdev ); 251 return -EINVAL; 252 } 253 254 /* Look in cache for a matching entry */ 255 cached = ib_find_path_cache_entry ( ibdev, gid ); 256 if ( cached && cached->path->av.lid ) { 257 /* Populated entry found */ 258 av->lid = cached->path->av.lid; 259 av->rate = cached->path->av.rate; 260 av->sl = cached->path->av.sl; 261 DBGC2 ( ibdev, "IBDEV %p cache hit for %08x:%08x:%08x:%08x\n", 262 ibdev, htonl ( gid->u.dwords[0] ), 263 htonl ( gid->u.dwords[1] ), htonl ( gid->u.dwords[2] ), 264 htonl ( gid->u.dwords[3] ) ); 265 return 0; 266 } 267 DBGC ( ibdev, "IBDEV %p cache miss for %08x:%08x:%08x:%08x%s\n", 268 ibdev, htonl ( gid->u.dwords[0] ), htonl ( gid->u.dwords[1] ), 269 htonl ( gid->u.dwords[2] ), htonl ( gid->u.dwords[3] ), 270 ( cached ? " (in progress)" : "" ) ); 271 272 /* If lookup is already in progress, do nothing */ 273 if ( cached ) 274 return -ENOENT; 275 276 /* Locate a new cache entry to use */ 277 cache_idx = ( (ib_path_cache_idx++) % IB_NUM_CACHED_PATHS ); 278 cached = &ib_path_cache[cache_idx]; 279 280 /* Destroy the old cache entry */ 281 if ( cached->path ) 282 ib_destroy_path ( ibdev, cached->path ); 283 memset ( cached, 0, sizeof ( *cached ) ); 284 285 /* Create new path */ 286 cached->path = ib_create_path ( ibdev, av, &ib_cached_path_op ); 287 if ( ! cached->path ) { 288 DBGC ( ibdev, "IBDEV %p could not create path\n", 289 ibdev ); 290 return -ENOMEM; 291 } 292 ib_path_set_ownerdata ( cached->path, cached ); 293 294 /* Not found yet */ 295 return -ENOENT; 296 } 297
